diff --git a/.gitignore b/.gitignore index 992ec7f2e..c7bdc0656 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,8 @@ tools/ .nuget/credprovider .nuget/.marker.v* nuget.exe +functionaltests.*.xml +AssemblyInfo.g.cs # Roslyn cache directories *.ide/ @@ -206,3 +208,8 @@ AssemblyInfo.*.cs /tests/Validation.Common.Tests/Validation.Common.Tests.nuget.props /tests/Validation.Common.Tests/Validation.Common.Tests.nuget.targets /tests/Validation.Helper.Tests/Validation.Helper.Tests.nuget.targets +/tests/packages +/tests/CatalogMetadataTests/CatalogMetadataTests.nuget.props +*.lock.json +/src/V3PerPackage/Settings.json +artifacts/* \ No newline at end of file diff --git a/.nuget/packages.config b/.nuget/packages.config index bb7daf160..eaca3ffaf 100644 --- a/.nuget/packages.config +++ b/.nuget/packages.config @@ -1,6 +1,8 @@ + + \ No newline at end of file diff --git a/NuGet.Jobs.sln b/NuGet.Jobs.sln index 4ba1d9ec4..7ebeda871 100644 --- a/NuGet.Jobs.sln +++ b/NuGet.Jobs.sln @@ -3,6 +3,12 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 VisualStudioVersion = 16.0.29722.177 MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Jobs", "Jobs", "{C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Services", "Services", "{97E23323-BA7A-48F0-A578-858B82B6D8FB}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Packages", "Packages", "{5DE01C58-D5F7-482F-8256-A8333064384C}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Common", "src\NuGet.Jobs.Common\NuGet.Jobs.Common.csproj", "{4B4B1EFB-8F33-42E6-B79F-54E7F3293D31}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{05F997A5-CD3A-4C60-9CCB-D630542EDA48}" @@ -21,6 +27,46 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Local.testsettings = Local.testsettings EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Metadata.Catalog", "src\Catalog\NuGet.Services.Metadata.Catalog.csproj", "{E97F23B8-ECB0-4AFA-B00C-015C39395FEF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CatalogTests", "tests\CatalogTests\CatalogTests.csproj", "{4D0B6BAB-5A33-4A7F-B007-93194FC2E2E3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Ng", "src\Ng\Ng.csproj", "{5234D86F-2C0E-4181-AAB7-BBDA3253B4E1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NgTests", "tests\NgTests\NgTests.csproj", "{05C1C78A-9966-4922-9065-A099023E7366}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.ApplicationInsights.Owin", "src\NuGet.ApplicationInsights.Owin\NuGet.ApplicationInsights.Owin.csproj", "{717E9A81-75C5-418E-92ED-18CAC55BC345}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.Metadata.Catalog.Monitoring", "src\NuGet.Services.Metadata.Catalog.Monitoring\NuGet.Services.Metadata.Catalog.Monitoring.csproj", "{1745A383-D0BE-484B-81EB-27B20F6AC6C5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CatalogMetadataTests", "tests\CatalogMetadataTests\CatalogMetadataTests.csproj", "{34AABA7F-1FF7-4F4B-B1DB-D07AD4505DA4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.AzureSearch", "src\NuGet.Services.AzureSearch\NuGet.Services.AzureSearch.csproj", "{1A53FE3D-8041-4773-942F-D73AEF5B82B2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.AzureSearch.Tests", "tests\NuGet.Services.AzureSearch.Tests\NuGet.Services.AzureSearch.Tests.csproj", "{6A9C3802-A2A2-49CF-87BD-C1303533B846}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Db2AzureSearch", "src\NuGet.Jobs.Db2AzureSearch\NuGet.Jobs.Db2AzureSearch.csproj", "{209B1B7F-1C5C-41EC-B6A6-E01FD9C86E26}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Catalog2AzureSearch", "src\NuGet.Jobs.Catalog2AzureSearch\NuGet.Jobs.Catalog2AzureSearch.csproj", "{F591130F-181A-4C53-8025-4390F46BD51D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Protocol.Catalog", "src\NuGet.Protocol.Catalog\NuGet.Protocol.Catalog.csproj", "{D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Protocol.Catalog.Tests", "tests\NuGet.Protocol.Catalog.Tests\NuGet.Protocol.Catalog.Tests.csproj", "{1F3BC053-796C-4A35-88F4-955A0F142197}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.SearchService", "src\NuGet.Services.SearchService\NuGet.Services.SearchService.csproj", "{DD089AB9-6AB3-4ACA-8D63-C95A7935B2A7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.SearchService.Tests", "tests\NuGet.Services.SearchService.Tests\NuGet.Services.SearchService.Tests.csproj", "{F009209D-A663-45E1-87E8-158569A0F097}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Auxiliary2AzureSearch", "src\NuGet.Jobs.Auxiliary2AzureSearch\NuGet.Jobs.Auxiliary2AzureSearch.csproj", "{7E6903A4-DBE1-444E-A8E3-C1DBB58243E0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Catalog2Registration", "src\NuGet.Jobs.Catalog2Registration\NuGet.Jobs.Catalog2Registration.csproj", "{5ABE8807-2209-4948-9FC5-1980A507C47A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.V3", "src\NuGet.Services.V3\NuGet.Services.V3.csproj", "{C3F9A738-9759-4B2B-A50D-6507B28A659B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Jobs.Catalog2Registration.Tests", "tests\NuGet.Jobs.Catalog2Registration.Tests\NuGet.Jobs.Catalog2Registration.Tests.csproj", "{296703A3-67BA-4876-8C1D-ACE13DF901EF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.V3.Tests", "tests\NuGet.Services.V3.Tests\NuGet.Services.V3.Tests.csproj", "{CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Stats.CollectAzureCdnLogs", "src\Stats.CollectAzureCdnLogs\Stats.CollectAzureCdnLogs.csproj", "{664CA8BB-BCF5-432C-AF68-9F97D308E623}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests.Stats.CollectAzureCdnLogs", "tests\Tests.Stats.CollectAzureCdnLogs\Tests.Stats.CollectAzureCdnLogs.csproj", "{578BCF9F-673B-4F18-8095-3B69A3D946DD}" @@ -49,8 +95,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Gallery", "Gallery", "{8872 EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Gallery.CredentialExpiration", "src\Gallery.CredentialExpiration\Gallery.CredentialExpiration.csproj", "{FA8C7905-985F-4919-AAA9-4B9A252F4977}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SupportRequests", "SupportRequests", "{BEC3DF4D-9A04-42C8-8B4F-D42750202B4D}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.SupportRequests.Notifications", "src\NuGet.SupportRequests.Notifications\NuGet.SupportRequests.Notifications.csproj", "{12719498-B87E-4E92-8C2B-30046393CF85}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Gallery.Maintenance", "src\Gallery.Maintenance\Gallery.Maintenance.csproj", "{EFF021CA-1BF4-4C09-BFB8-D314EAAD24D2}" @@ -145,6 +189,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Ad-Hoc Tools", "Ad-Hoc Tool EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SplitLargeFiles.Tests", "tests\SplitLargeFiles.Tests\SplitLargeFiles.Tests.csproj", "{DD70035C-1BAB-4EFF-B270-EF45F7473C22}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "V3", "V3", "{0A4A2A3F-8887-431E-B1B3-E9DF9B155BA6}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -159,6 +205,86 @@ Global {A08F0185-4DA3-4D5B-BF0C-B178596D5789}.Debug|Any CPU.Build.0 = Debug|Any CPU {A08F0185-4DA3-4D5B-BF0C-B178596D5789}.Release|Any CPU.ActiveCfg = Release|Any CPU {A08F0185-4DA3-4D5B-BF0C-B178596D5789}.Release|Any CPU.Build.0 = Release|Any CPU + {E97F23B8-ECB0-4AFA-B00C-015C39395FEF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E97F23B8-ECB0-4AFA-B00C-015C39395FEF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E97F23B8-ECB0-4AFA-B00C-015C39395FEF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E97F23B8-ECB0-4AFA-B00C-015C39395FEF}.Release|Any CPU.Build.0 = Release|Any CPU + {4D0B6BAB-5A33-4A7F-B007-93194FC2E2E3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4D0B6BAB-5A33-4A7F-B007-93194FC2E2E3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4D0B6BAB-5A33-4A7F-B007-93194FC2E2E3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4D0B6BAB-5A33-4A7F-B007-93194FC2E2E3}.Release|Any CPU.Build.0 = Release|Any CPU + {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1}.Release|Any CPU.Build.0 = Release|Any CPU + {05C1C78A-9966-4922-9065-A099023E7366}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {05C1C78A-9966-4922-9065-A099023E7366}.Debug|Any CPU.Build.0 = Debug|Any CPU + {05C1C78A-9966-4922-9065-A099023E7366}.Release|Any CPU.ActiveCfg = Release|Any CPU + {05C1C78A-9966-4922-9065-A099023E7366}.Release|Any CPU.Build.0 = Release|Any CPU + {717E9A81-75C5-418E-92ED-18CAC55BC345}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {717E9A81-75C5-418E-92ED-18CAC55BC345}.Debug|Any CPU.Build.0 = Debug|Any CPU + {717E9A81-75C5-418E-92ED-18CAC55BC345}.Release|Any CPU.ActiveCfg = Release|Any CPU + {717E9A81-75C5-418E-92ED-18CAC55BC345}.Release|Any CPU.Build.0 = Release|Any CPU + {1745A383-D0BE-484B-81EB-27B20F6AC6C5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1745A383-D0BE-484B-81EB-27B20F6AC6C5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1745A383-D0BE-484B-81EB-27B20F6AC6C5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1745A383-D0BE-484B-81EB-27B20F6AC6C5}.Release|Any CPU.Build.0 = Release|Any CPU + {34AABA7F-1FF7-4F4B-B1DB-D07AD4505DA4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {34AABA7F-1FF7-4F4B-B1DB-D07AD4505DA4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {34AABA7F-1FF7-4F4B-B1DB-D07AD4505DA4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {34AABA7F-1FF7-4F4B-B1DB-D07AD4505DA4}.Release|Any CPU.Build.0 = Release|Any CPU + {1A53FE3D-8041-4773-942F-D73AEF5B82B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1A53FE3D-8041-4773-942F-D73AEF5B82B2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1A53FE3D-8041-4773-942F-D73AEF5B82B2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1A53FE3D-8041-4773-942F-D73AEF5B82B2}.Release|Any CPU.Build.0 = Release|Any CPU + {6A9C3802-A2A2-49CF-87BD-C1303533B846}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6A9C3802-A2A2-49CF-87BD-C1303533B846}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6A9C3802-A2A2-49CF-87BD-C1303533B846}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6A9C3802-A2A2-49CF-87BD-C1303533B846}.Release|Any CPU.Build.0 = Release|Any CPU + {209B1B7F-1C5C-41EC-B6A6-E01FD9C86E26}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {209B1B7F-1C5C-41EC-B6A6-E01FD9C86E26}.Debug|Any CPU.Build.0 = Debug|Any CPU + {209B1B7F-1C5C-41EC-B6A6-E01FD9C86E26}.Release|Any CPU.ActiveCfg = Release|Any CPU + {209B1B7F-1C5C-41EC-B6A6-E01FD9C86E26}.Release|Any CPU.Build.0 = Release|Any CPU + {F591130F-181A-4C53-8025-4390F46BD51D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F591130F-181A-4C53-8025-4390F46BD51D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F591130F-181A-4C53-8025-4390F46BD51D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F591130F-181A-4C53-8025-4390F46BD51D}.Release|Any CPU.Build.0 = Release|Any CPU + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C}.Release|Any CPU.Build.0 = Release|Any CPU + {1F3BC053-796C-4A35-88F4-955A0F142197}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1F3BC053-796C-4A35-88F4-955A0F142197}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1F3BC053-796C-4A35-88F4-955A0F142197}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1F3BC053-796C-4A35-88F4-955A0F142197}.Release|Any CPU.Build.0 = Release|Any CPU + {DD089AB9-6AB3-4ACA-8D63-C95A7935B2A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DD089AB9-6AB3-4ACA-8D63-C95A7935B2A7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DD089AB9-6AB3-4ACA-8D63-C95A7935B2A7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DD089AB9-6AB3-4ACA-8D63-C95A7935B2A7}.Release|Any CPU.Build.0 = Release|Any CPU + {F009209D-A663-45E1-87E8-158569A0F097}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F009209D-A663-45E1-87E8-158569A0F097}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F009209D-A663-45E1-87E8-158569A0F097}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F009209D-A663-45E1-87E8-158569A0F097}.Release|Any CPU.Build.0 = Release|Any CPU + {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0}.Release|Any CPU.Build.0 = Release|Any CPU + {5ABE8807-2209-4948-9FC5-1980A507C47A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5ABE8807-2209-4948-9FC5-1980A507C47A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5ABE8807-2209-4948-9FC5-1980A507C47A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5ABE8807-2209-4948-9FC5-1980A507C47A}.Release|Any CPU.Build.0 = Release|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B}.Release|Any CPU.Build.0 = Release|Any CPU + {296703A3-67BA-4876-8C1D-ACE13DF901EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {296703A3-67BA-4876-8C1D-ACE13DF901EF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {296703A3-67BA-4876-8C1D-ACE13DF901EF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {296703A3-67BA-4876-8C1D-ACE13DF901EF}.Release|Any CPU.Build.0 = Release|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A}.Release|Any CPU.Build.0 = Release|Any CPU {664CA8BB-BCF5-432C-AF68-9F97D308E623}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {664CA8BB-BCF5-432C-AF68-9F97D308E623}.Debug|Any CPU.Build.0 = Debug|Any CPU {664CA8BB-BCF5-432C-AF68-9F97D308E623}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -390,8 +516,31 @@ Global HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution + {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4} = {0A4A2A3F-8887-431E-B1B3-E9DF9B155BA6} + {97E23323-BA7A-48F0-A578-858B82B6D8FB} = {0A4A2A3F-8887-431E-B1B3-E9DF9B155BA6} + {5DE01C58-D5F7-482F-8256-A8333064384C} = {0A4A2A3F-8887-431E-B1B3-E9DF9B155BA6} {A08F0185-4DA3-4D5B-BF0C-B178596D5789} = {FA5644B5-4F08-43F6-86B3-039374312A47} {6078A997-EDEF-4F84-9E9C-163EB1E7454D} = {6A776396-02B1-475D-A104-26940ADB04AB} + {E97F23B8-ECB0-4AFA-B00C-015C39395FEF} = {5DE01C58-D5F7-482F-8256-A8333064384C} + {4D0B6BAB-5A33-4A7F-B007-93194FC2E2E3} = {6A776396-02B1-475D-A104-26940ADB04AB} + {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4} + {05C1C78A-9966-4922-9065-A099023E7366} = {6A776396-02B1-475D-A104-26940ADB04AB} + {717E9A81-75C5-418E-92ED-18CAC55BC345} = {5DE01C58-D5F7-482F-8256-A8333064384C} + {1745A383-D0BE-484B-81EB-27B20F6AC6C5} = {5DE01C58-D5F7-482F-8256-A8333064384C} + {34AABA7F-1FF7-4F4B-B1DB-D07AD4505DA4} = {6A776396-02B1-475D-A104-26940ADB04AB} + {1A53FE3D-8041-4773-942F-D73AEF5B82B2} = {5DE01C58-D5F7-482F-8256-A8333064384C} + {6A9C3802-A2A2-49CF-87BD-C1303533B846} = {6A776396-02B1-475D-A104-26940ADB04AB} + {209B1B7F-1C5C-41EC-B6A6-E01FD9C86E26} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4} + {F591130F-181A-4C53-8025-4390F46BD51D} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4} + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} = {5DE01C58-D5F7-482F-8256-A8333064384C} + {1F3BC053-796C-4A35-88F4-955A0F142197} = {6A776396-02B1-475D-A104-26940ADB04AB} + {DD089AB9-6AB3-4ACA-8D63-C95A7935B2A7} = {97E23323-BA7A-48F0-A578-858B82B6D8FB} + {F009209D-A663-45E1-87E8-158569A0F097} = {6A776396-02B1-475D-A104-26940ADB04AB} + {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4} + {5ABE8807-2209-4948-9FC5-1980A507C47A} = {C86C6DEE-84E1-4E4E-8868-6755D7A8E0E4} + {C3F9A738-9759-4B2B-A50D-6507B28A659B} = {5DE01C58-D5F7-482F-8256-A8333064384C} + {296703A3-67BA-4876-8C1D-ACE13DF901EF} = {6A776396-02B1-475D-A104-26940ADB04AB} + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A} = {6A776396-02B1-475D-A104-26940ADB04AB} {664CA8BB-BCF5-432C-AF68-9F97D308E623} = {B9D03824-A9CA-43AC-86D6-8BB399B9A228} {578BCF9F-673B-4F18-8095-3B69A3D946DD} = {6A776396-02B1-475D-A104-26940ADB04AB} {F72C31A7-424D-48C6-924C-EBFD4BE0918B} = {B9D03824-A9CA-43AC-86D6-8BB399B9A228} @@ -403,10 +552,10 @@ Global {B5C01B7A-933D-483E-AF07-6AA266B0EB49} = {B9D03824-A9CA-43AC-86D6-8BB399B9A228} {3E0A20C8-C6D2-4762-955D-C7BF35C2C9A7} = {B9D03824-A9CA-43AC-86D6-8BB399B9A228} {FA8C7905-985F-4919-AAA9-4B9A252F4977} = {88725659-D5F8-49F9-9B7E-D87C5B9917D7} - {12719498-B87E-4E92-8C2B-30046393CF85} = {BEC3DF4D-9A04-42C8-8B4F-D42750202B4D} + {12719498-B87E-4E92-8C2B-30046393CF85} = {FA5644B5-4F08-43F6-86B3-039374312A47} {EFF021CA-1BF4-4C09-BFB8-D314EAAD24D2} = {88725659-D5F8-49F9-9B7E-D87C5B9917D7} {A07F7D0C-F269-43D5-A812-3ABC47090885} = {FA5644B5-4F08-43F6-86B3-039374312A47} - {BC9EA7CE-AD21-4D17-B581-F8ED8CBD7191} = {FA5644B5-4F08-43F6-86B3-039374312A47} + {BC9EA7CE-AD21-4D17-B581-F8ED8CBD7191} = {97E23323-BA7A-48F0-A578-858B82B6D8FB} {147A757D-864B-4C74-B8CF-14DFF9793605} = {6A776396-02B1-475D-A104-26940ADB04AB} {FC0CEF12-D501-46D1-B1BF-D4134BD8D478} = {B9D03824-A9CA-43AC-86D6-8BB399B9A228} {0C887292-C5AB-4107-946C-A53B18A38D22} = {6A776396-02B1-475D-A104-26940ADB04AB} diff --git a/README.md b/README.md index 507081ce7..a8cd75730 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ NuGet.Jobs -============== +========== + +This repo contains nuget.org's implementation of the [NuGet V3 API](https://docs.microsoft.com/en-us/nuget/api/overview) +as well as many other back-end jobs for the operation of nuget.org. 1. Each job would be an exe with 2 main classes Program and Job 2. Program.Main should simply do the following and nothing more @@ -20,6 +23,14 @@ NuGet.Jobs 7. Also, add settings.job file to mark the job as singleton, if the job will be run as a webjob, and it be a continuously running singleton +## Feedback + +If you're having trouble with the NuGet.org Website, file a bug on the [NuGet Gallery Issue Tracker](https://github.com/nuget/NuGetGallery/issues). + +If you're having trouble with the NuGet client tools (the Visual Studio extension, NuGet.exe command line tool, etc.), file a bug on [NuGet Home](https://github.com/nuget/home/issues). + +Check out the [contributing](http://docs.nuget.org/contribute) page to see the best places to log issues and start discussions. The [NuGet Home](https://github.com/NuGet/Home) repo provides an overview of the different NuGet projects available. + Open Source Code of Conduct =================== This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/build.ps1 b/build.ps1 index 29bbcf539..65fc9b4e4 100644 --- a/build.ps1 +++ b/build.ps1 @@ -87,25 +87,36 @@ Invoke-BuildStep 'Clearing artifacts' { Clear-Artifacts } ` Invoke-BuildStep 'Set version metadata in AssemblyInfo.cs' { ` $versionMetadata = - "$PSScriptRoot\src\CopyAzureContainer\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\NuGetCDNRedirect\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\NuGet.Services.Validation.Orchestrator\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\NuGet.Services.Revalidate\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\Stats.CollectAzureChinaCDNLogs\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\Validation.PackageSigning.ProcessSignature\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\Validation.PackageSigning.ValidateCertificate\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\Validation.PackageSigning.RevalidateCertificate\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\Validation.Common.Job\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\Validation.ScanAndSign.Core\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\PackageLagMonitor\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\StatusAggregator\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\Validation.Symbols.Core\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\Stats.CDNLogsSanitizer\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\NuGet.Jobs.GitHubIndexer\Properties\AssemblyInfo.g.cs", - "$PSScriptRoot\src\SplitLargeFiles\Properties\AssemblyInfo.g.cs" + "src\CopyAzureContainer\Properties\AssemblyInfo.g.cs", + "src\NuGetCDNRedirect\Properties\AssemblyInfo.g.cs", + "src\NuGet.Services.Validation.Orchestrator\Properties\AssemblyInfo.g.cs", + "src\NuGet.Services.Revalidate\Properties\AssemblyInfo.g.cs", + "src\Stats.CollectAzureChinaCDNLogs\Properties\AssemblyInfo.g.cs", + "src\Validation.PackageSigning.ProcessSignature\Properties\AssemblyInfo.g.cs", + "src\Validation.PackageSigning.ValidateCertificate\Properties\AssemblyInfo.g.cs", + "src\Validation.PackageSigning.RevalidateCertificate\Properties\AssemblyInfo.g.cs", + "src\Validation.Common.Job\Properties\AssemblyInfo.g.cs", + "src\Validation.ScanAndSign.Core\Properties\AssemblyInfo.g.cs", + "src\PackageLagMonitor\Properties\AssemblyInfo.g.cs", + "src\StatusAggregator\Properties\AssemblyInfo.g.cs", + "src\Validation.Symbols.Core\Properties\AssemblyInfo.g.cs", + "src\Stats.CDNLogsSanitizer\Properties\AssemblyInfo.g.cs", + "src\NuGet.Jobs.GitHubIndexer\Properties\AssemblyInfo.g.cs", + "src\SplitLargeFiles\Properties\AssemblyInfo.g.cs", + "src\Catalog\Properties\AssemblyInfo.g.cs", + "src\NuGet.ApplicationInsights.Owin\Properties\AssemblyInfo.g.cs", + "src\Ng\Properties\AssemblyInfo.g.cs", + "src\NuGet.Services.Metadata.Catalog.Monitoring\Properties\AssemblyInfo.g.cs", + "src\NuGet.Protocol.Catalog\Properties\AssemblyInfo.g.cs", + "src\NuGet.Services.AzureSearch\Properties\AssemblyInfo.g.cs", + "src\NuGet.Jobs.Db2AzureSearch\Properties\AssemblyInfo.g.cs", + "src\NuGet.Jobs.Catalog2AzureSearch\Properties\AssemblyInfo.g.cs", + "src\NuGet.Services.SearchService\Properties\AssemblyInfo.g.cs", + "src\NuGet.Jobs.Auxiliary2AzureSearch\Properties\AssemblyInfo.g.cs", + "src\NuGet.Jobs.Catalog2Registration\Properties\AssemblyInfo.g.cs" $versionMetadata | ForEach-Object { - Set-VersionInfo -Path $_ -Version $SimpleVersion -Branch $Branch -Commit $CommitSHA + Set-VersionInfo -Path (Join-Path $PSScriptRoot $_) -Version $SimpleVersion -Branch $Branch -Commit $CommitSHA } } ` -ev +BuildErrors @@ -122,6 +133,12 @@ Invoke-BuildStep 'Building solution' { -args $Configuration, $BuildNumber, (Join-Path $PSScriptRoot "NuGet.Jobs.sln"), $SkipRestore ` -ev +BuildErrors +Invoke-BuildStep 'Building functional test solution' { + $SolutionPath = Join-Path $PSScriptRoot "tests\NuGetServicesMetadata.FunctionalTests.sln" + Build-Solution $Configuration $BuildNumber -MSBuildVersion "$msBuildVersion" $SolutionPath -SkipRestore:$SkipRestore ` + } ` + -ev +BuildErrors + Invoke-BuildStep 'Signing the binaries' { Sign-Binaries -Configuration $Configuration -BuildNumber $BuildNumber -MSBuildVersion "15" ` } ` @@ -135,40 +152,57 @@ Invoke-BuildStep 'Creating artifacts' { # We need symbols published for those, too. All other packages are deployment ones and # don't need to be shared, hence no need for symbols for them $CsprojProjects = - "src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj", - "src/Validation.Common.Job/Validation.Common.Job.csproj", - "src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj", - "src/Validation.Symbols.Core/Validation.Symbols.Core.csproj" + "src\NuGet.Jobs.Common\NuGet.Jobs.Common.csproj", + "src\Validation.Common.Job\Validation.Common.Job.csproj", + "src\Validation.ScanAndSign.Core\Validation.ScanAndSign.Core.csproj", + "src\Validation.Symbols.Core\Validation.Symbols.Core.csproj", + "src\Catalog\NuGet.Services.Metadata.Catalog.csproj", + "src\NuGet.ApplicationInsights.Owin\NuGet.ApplicationInsights.Owin.csproj", + "src\NuGet.Services.Metadata.Catalog.Monitoring\NuGet.Services.Metadata.Catalog.Monitoring.csproj", + "src\NuGet.Protocol.Catalog\NuGet.Protocol.Catalog.csproj", + "src\NuGet.Services.AzureSearch\NuGet.Services.AzureSearch.csproj" $CsprojProjects | ForEach-Object { New-ProjectPackage (Join-Path $PSScriptRoot $_) -Configuration $Configuration -BuildNumber $BuildNumber -Version $SemanticVersion -Branch $Branch -Symbols } $NuspecProjects = ` - "src/Stats.CollectAzureCdnLogs/Stats.CollectAzureCdnLogs.csproj", ` - "src/Stats.AggregateCdnDownloadsInGallery/Stats.AggregateCdnDownloadsInGallery.csproj", ` - "src/Stats.ImportAzureCdnStatistics/Stats.ImportAzureCdnStatistics.csproj", ` - "src/Stats.CreateAzureCdnWarehouseReports/Stats.CreateAzureCdnWarehouseReports.csproj", ` - "src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj", ` - "src/Gallery.Maintenance/Gallery.Maintenance.nuspec", ` - "src/ArchivePackages/ArchivePackages.csproj", ` - "src/Stats.RollUpDownloadFacts/Stats.RollUpDownloadFacts.csproj", ` - "src/NuGet.SupportRequests.Notifications/NuGet.SupportRequests.Notifications.csproj", ` - "src/CopyAzureContainer/CopyAzureContainer.csproj", ` - "src/NuGet.Services.Validation.Orchestrator/Validation.Orchestrator.nuspec", ` - "src/NuGet.Services.Validation.Orchestrator/Validation.SymbolsOrchestrator.nuspec", ` - "src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj", ` - "src/Stats.CollectAzureChinaCDNLogs/Stats.CollectAzureChinaCDNLogs.csproj", ` - "src/Validation.PackageSigning.ProcessSignature/Validation.PackageSigning.ProcessSignature.csproj", ` - "src/Validation.PackageSigning.ValidateCertificate/Validation.PackageSigning.ValidateCertificate.csproj", ` - "src/Validation.PackageSigning.RevalidateCertificate/Validation.PackageSigning.RevalidateCertificate.csproj", ` - "src/PackageLagMonitor/Monitoring.PackageLag.csproj", ` - "src/StatusAggregator/StatusAggregator.csproj", ` - "src/Validation.Symbols.Core/Validation.Symbols.Core.csproj", ` - "src/Validation.Symbols/Validation.Symbols.Job.csproj", ` - "src/Stats.CDNLogsSanitizer/Stats.CDNLogsSanitizer.csproj", ` - "src/NuGet.Jobs.GitHubIndexer/NuGet.Jobs.GitHubIndexer.nuspec", ` - "src/SplitLargeFiles/SplitLargeFiles.nuspec" + "src\Stats.CollectAzureCdnLogs\Stats.CollectAzureCdnLogs.csproj", ` + "src\Stats.AggregateCdnDownloadsInGallery\Stats.AggregateCdnDownloadsInGallery.csproj", ` + "src\Stats.ImportAzureCdnStatistics\Stats.ImportAzureCdnStatistics.csproj", ` + "src\Stats.CreateAzureCdnWarehouseReports\Stats.CreateAzureCdnWarehouseReports.csproj", ` + "src\Gallery.CredentialExpiration\Gallery.CredentialExpiration.csproj", ` + "src\Gallery.Maintenance\Gallery.Maintenance.nuspec", ` + "src\ArchivePackages\ArchivePackages.csproj", ` + "src\Stats.RollUpDownloadFacts\Stats.RollUpDownloadFacts.csproj", ` + "src\NuGet.SupportRequests.Notifications\NuGet.SupportRequests.Notifications.csproj", ` + "src\CopyAzureContainer\CopyAzureContainer.csproj", ` + "src\NuGet.Services.Validation.Orchestrator\Validation.Orchestrator.nuspec", ` + "src\NuGet.Services.Validation.Orchestrator\Validation.SymbolsOrchestrator.nuspec", ` + "src\NuGet.Services.Revalidate\NuGet.Services.Revalidate.csproj", ` + "src\Stats.CollectAzureChinaCDNLogs\Stats.CollectAzureChinaCDNLogs.csproj", ` + "src\Validation.PackageSigning.ProcessSignature\Validation.PackageSigning.ProcessSignature.csproj", ` + "src\Validation.PackageSigning.ValidateCertificate\Validation.PackageSigning.ValidateCertificate.csproj", ` + "src\Validation.PackageSigning.RevalidateCertificate\Validation.PackageSigning.RevalidateCertificate.csproj", ` + "src\PackageLagMonitor\Monitoring.PackageLag.csproj", ` + "src\StatusAggregator\StatusAggregator.csproj", ` + "src\Validation.Symbols.Core\Validation.Symbols.Core.csproj", ` + "src\Validation.Symbols\Validation.Symbols.Job.csproj", ` + "src\Stats.CDNLogsSanitizer\Stats.CDNLogsSanitizer.csproj", ` + "src\NuGet.Jobs.GitHubIndexer\NuGet.Jobs.GitHubIndexer.nuspec", ` + "src\SplitLargeFiles\SplitLargeFiles.nuspec", ` + "src\Ng\Catalog2Dnx.nuspec", ` + "src\Ng\Catalog2icon.nuspec", ` + "src\Ng\Catalog2Monitoring.nuspec", ` + "src\Ng\Db2Catalog.nuspec", ` + "src\Ng\Db2Monitoring.nuspec", ` + "src\Ng\Monitoring2Monitoring.nuspec", ` + "src\Ng\MonitoringProcessor.nuspec", ` + "src\Ng\Ng.Operations.nuspec", ` + "src\NuGet.Jobs.Db2AzureSearch\NuGet.Jobs.Db2AzureSearch.nuspec", ` + "src\NuGet.Jobs.Catalog2AzureSearch\NuGet.Jobs.Catalog2AzureSearch.nuspec", ` + "src\NuGet.Jobs.Auxiliary2AzureSearch\NuGet.Jobs.Auxiliary2AzureSearch.nuspec", ` + "src\NuGet.Jobs.Catalog2Registration\NuGet.Jobs.Catalog2Registration.nuspec" Foreach ($Project in $NuspecProjects) { New-Package (Join-Path $PSScriptRoot "$Project") -Configuration $Configuration -BuildNumber $BuildNumber -Version $SemanticVersion -Branch $Branch -MSBuildVersion "$msBuildVersion" diff --git a/docs/Azure-Search-indexes.md b/docs/Azure-Search-indexes.md new file mode 100644 index 000000000..f4dc7a86e --- /dev/null +++ b/docs/Azure-Search-indexes.md @@ -0,0 +1,111 @@ +# Azure Search indexes + +**Subsystem: Search 🔎** + +The search subsystem heavily depends on Azure Search for storing package metadata and performing package queries. Within +a single Azure Search resource, there can be multiple indexes. An index is simply a collection of documents with a +common schema. For the NuGet search subsystem, there are two indexes expected in each Azure Search resource: + +- [`search-XXX`](#search-index) - this is the "search" index which contains documents for *discovery* queries +- [`hijack-XXX`](#hijack-index) - this is the "hijack" index which contains documents for *metadata lookup* queries + +## Search index + +The search index is designed to fulfill queries for package discovery. This is likely the scenario you would think about +first when you imagine how package search would work. It's optimized for searching package metadata field by one or more +keywords and has a scoring profile that returns the most relevant package first. + +This index has up to four documents per package ID. Each of the four ID-specific documents represents a different view +of available package versions. There are two factors for filtering in and out package versions: whether or not to +consider prerelease versions and whether or not to consider SemVer 2.0.0 versions. + +This may seem is a little strange at first, so it's best to consider an example. Consider a package +[`BaseTestPackage.SearchFilters`](https://www.nuget.org/packages/BaseTestPackage.SearchFilters) that has four versions: + +- `1.1.0` - stable, SemVer 1.0.0 +- `1.2.0-beta`, prerelease, SemVer 1.0.0 +- `1.3.0+metadata`, stable, SemVer 2.0.0 (due to build metadata) +- `1.4.0-delta.4`, prerelease, SemVer 2.0.0 (due to a dot in the prerelease label) + +As mentioned before there are up to four documents per package ID. In the case of the example package +`BaseTestPackage.SearchFilters`, there will be four documents, each with a different set of versions included in the +document. + +- Stable + SemVer 1.0.0: contains only `1.1.0` ([example query](https://azuresearch-usnc.nuget.org/query?q=packageid:BaseTestPackage.SearchFilters)) +- Stable/Prerelease + SemVer 1.0.0: contains `1.1.0` and `1.2.0-beta` ([example query](https://azuresearch-usnc.nuget.org/query?q=packageid:BaseTestPackage.SearchFilters&prerelease=true)) +- Stable + SemVer 2.0.0: contains `1.1.0` and `1.3.0+metadata` ([example query](https://azuresearch-usnc.nuget.org/query?q=packageid:BaseTestPackage.SearchFilters&semVerLevel=2.0.0)) +- Stable/Prerelease + SemVer 2.0.0: contains all versions ([example query](https://azuresearch-usnc.nuget.org/query?q=packageid:BaseTestPackage.SearchFilters&prerelease=true&semVerLevel=2.0.0)) + +The four "flavors" of search documents per ID are referred to as **search filters**. + +The documents in the search index are identified (via the `key` property) by a unique string with the following format: + +``` +{sanitized lowercase ID}-{base64 lowercase ID}-{search filter} +``` + +The `sanitized lowercase ID` removes all characters from the package ID that are not acceptable for Azure Search +document keys, like dots and non-ASCII word characters (like Chinese characters). This component of the document key is +included for readability purposes only. + +The `base64 lowercase ID` is the base64 encoding of the package ID's bytes, encoded with UTF-8. This string is +guaranteed to be a 1:1 mapping with the lowercase package ID and is included for uniqueness. The +`HttpServerUtility.UrlTokenEncode` API is used for base64 encoding. + +The `search filter` has one of four values: + +- `Default` - Stable + SemVer 1.0.0 +- `IncludePrerelease` - Stable/Prerelease + SemVer 1.0.0 +- `IncludeSemVer2` - Stable + SemVer 2.0.0 +- `IncludePrereleaseAndSemVer2` - Stable/Prerelease + SemVer 2.0.0 + +For the package ID `BaseTestPackage.SearchFilters`, the Stable + 1.0.0 document key would be: + +``` +basetestpackage_searchfilters-YmFzZXRlc3RwYWNrYWdlLnNlYXJjaGZpbHRlcnM1-Default +``` + +Each document contains a variety of metadata fields originating from the latest version in the application version list +as well as a field listing all versions. See the +[`NuGet.Services.AzureSearch.SearchDocument.Full`](../src/NuGet.Services.AzureSearch/Models/SearchDocument.cs) class and +its inherited members for a full list of the fields. + +Unlisted package versions do not appear in the search index at all. + +## Hijack index + +The hijack index is used by the gallery to fulfill specific metadata lookup operations. For example, if a +customer is looking for metadata about all versions of the package ID `Newtonsoft.Json`, in certain cases the gallery +will query the search service for this metadata and the search service will use the hijack index to fetch the +data. + +This index has one document for every version of every package ID, whether it is unlisted or not. The search service +uses this index to find all versions of a package via the `ignoreFilter=true` parameter including, + +- unlisted packages ([example query](https://azuresearch-usnc.nuget.org/search/query?q=packageid:BaseTestPackage.Unlisted&ignoreFilter=true)) +- multiple versions of a single ID ([example query](https://azuresearch-usnc.nuget.org/search/query?q=packageid:BaseTestPackage.SearchFilters&ignoreFilter=true&semVerLevel=2.0.0)) + +The documents in the hijack index are identified (via the `key` property) by a unique string with the following format: + +``` +{sanitized ID/version}-{base64 ID/version} +``` + +The `sanitized ID/version` removes all characters from the `{lowercase package ID}/{lowercase, normalized version}` +that are not acceptable for Azure Search document keys, like dots and non-ASCII word characters (like Chinese +characters). This component of the document key is included for readability purposes only. + +The `base64 ID/version` is the base64 encoding of the previously mentioned concatenation of ID and version, encoded +with UTF-8. This string is guaranteed to be a 1:1 mapping with the lowercase package ID and version and is included +for uniqueness. The `HttpServerUtility.UrlTokenEncode` API is used for base64 encoding. + +For the package ID `BaseTestPackage.SearchFilters` and version `1.3.0+metadata`, the document key would be: + +``` +basetestpackage_searchfilters_1_3_0-YmFzZXRlc3RwYWNrYWdlLnNlYXJjaGZpbHRlcnMvMS4zLjA1 +``` + +Each document contains a variety of metadata fields originating from the latest version in the application version list +as well as a field listing all versions. See the +[`NuGet.Services.AzureSearch.HijackDocument.Full`](../src/NuGet.Services.AzureSearch/Models/HijackDocument.cs) class and +its inherited members for a full list of the fields. diff --git a/docs/Search-auxiliary-files.md b/docs/Search-auxiliary-files.md new file mode 100644 index 000000000..1aaad3c57 --- /dev/null +++ b/docs/Search-auxiliary-files.md @@ -0,0 +1,169 @@ +# Search auxiliary files + +**Subsystem: Search 🔎** + +Aside from metadata stored in the [Azure Search indexes](Azure-Search-indexes.md), there is data stored in Azure Blob +Storage for bookkeeping and performance reasons. These data files are called **auxiliary files**. The data files +mentioned here are those explicitly managed by the search subsystem. Other data files exist (manually created, +created by the statistics subsystem, etc.). Those will not be covered here but are mentioned in the job-specific +documentation that uses them as input. + +Each search auxiliary file is copied to the individual region that a [search service](../src/NuGet.Services.SearchService/README.md) +is deployed. For nuget.org, we run search in four regions, so there are four copies of each of these files. + +The search auxiliary files are: + + - [`downloads/downloads.v2.json`](#download-count-data) - total download count for every package version + - [`owners/owners.v2.json` and change history](#package-ownership-data) - owners for every package ID + - [`verified-packages/verified-packages.v1.json`](#verified-packages-data) - package IDs that are verified + - [`popularity-transfers/popularity-transfers.v1.json`](#popularity-transfer-data) - popularity transfers between package IDs + +## Download count data + +The `downloads/downloads.v2.json` file has the total download count for all package versions. The total download count +for a package ID as a whole can be calculated simply by adding all version download counts. + +The downloads data file looks like this: + +```json +{ + "Newtonsoft.Json": { + "8.0.3": 10508321, + "9.0.1": 55801938 + }, + "NuGet.Versioning": { + "5.6.0-preview.3.6558": 988, + "5.6.0": 10224 + } +} +``` + +The package ID and version keys are not guaranteed to have the original (author-intended) casing and should be treated +in a case insensitive manner. The version keys will always be normalized via [standard `NuGetVersion` normalization rules](https://docs.microsoft.com/en-us/nuget/concepts/package-versioning#normalized-version-numbers) +(e.g. no build metadata will appear, no leading zeroes, etc.). + +If a package ID or version does not exist in the data file, this only indicates that there is no download count data and +does not imply that the package ID or version does not exist on the package source. It is possible for package IDs or +versions that do not exist (perhaps due to deletion) to exist in the data file. + +The order of the IDs and versions in the file is undefined. + +This file has a "v2" in the file name because it is the second version of this data. The "v1" format is still produced +by the statistics subsystem and has a less friendly data format. + +The class for reading and writing this file to Blob Storage is [`DownloadDataClient`](../src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadDataClient.cs). + +## Package ownership data + +The `owners/owners.v2.json` file contains the owner information about all package IDs. Each time this file is updated, +the set of package IDs that changed is written to a "change history" file with a path pattern like +`owners/changes/TIMESTAMP.json`. + +The class for reading and writing these files to Blob Storage is [`OwnerDataClient`](../src/NuGet.Services.AzureSearch/AuxiliaryFiles/OwnerDataClient.cs). + +### `owners/owners.v2.json` + +The owners data file looks like this: + +```json +{ + "Newtonsoft.Json": [ + "dotnetfoundation", + "jamesnk", + "newtonsoft" + ], + "NuGet.Versioning": [ + "Microsoft", + "nuget" + ] +} +``` + +The package ID key is not guaranteed to have the original (author-intended) casing and should be treated +in a case insensitive manner. The owner values will have the same casing that is shown on NuGetGallery but should be +treated in a case insensitive manner. + +If a package ID does not exist in the data file, this indicates that the package ID has no owners (a possible but +relatively rare scenario for NuGetGallery). It is possible for a package ID with no versions to appear in this file. + +The order of the IDs and owner usernames in the file is case insensitive ascending lexicographical order. + +This file has a "v2" in the file name because it is the second version of this data. The "v1" format was deprecated when +nuget.org moved from a Lucene-based search service to Azure Search. The "v1" format had a less friendly data format. + +### Change history + +The change history files do not contain owner usernames for GDPR reasons but mention all of the package IDs that had +ownership changes since the last time that the `owners.v2.json` file was generated. If a package ID is not mentioned in +a file, that means that there were no ownership changes in the time window. An ownership change is defined as one or +more owners being added or removed from the set of owners for that package ID. + +Each change history data file has a file name with timestamp format `yyyy-MM-dd-HH-mm-ss-FFFFFFF` (UTC) and a file +extension of `.json`. + +The files look like this: + +```json +[ + "Newtonsoft.Json", + "NuGet.Versioning" +] +``` + +By processing the files in order of their timestamp file name, a rough log of ownership changes can be produced. These +files are currently not read by any job and are produced for future investigative purposes. + +The package ID key is not guaranteed to have the original (author-intended) casing and should be treated +in a case insensitive manner. + +The order of the package IDs in the file is undefined. + +## Verified packages data + +The `verified-packages/verified-packages.v1.json` data file contains all package IDs that are considered verified by the [prefix reservation feature](https://docs.microsoft.com/en-us/nuget/nuget-org/id-prefix-reservation). This essentially defines the verified checkmark icon in the search UIs. + +The data file looks like this: + +```json +[ + "Newtonsoft.Json", + "NuGet.Versioning" +] +``` + +If a package ID is in the file, then it is verified. The package ID is not guaranteed to have the original +(author-intended) casing and should be treated in a case insensitive manner. + +The order of the package IDs is undefined. + +The class for reading and writing this file to Blob Storage is [`VerifiedPackagesDataClient`](../src/NuGet.Services.AzureSearch/AuxiliaryFiles/VerifiedPackagesDataClient.cs). + +## Popularity transfer data + +The `popularity-transfers/popularity-transfers.v1.json` data file has a mapping of all package IDs that have +transferred their popularity to one or more other packages. + +The data file looks like this: + +```json +{ + "OldPackageA": [ + "NewPackage1", + "NewPackage2" + ], + "OldPackageB": [ + "NewPackage3" + ] +} +``` + +For each key-value pair, the package ID key has its popularity transferred to the package ID values. The implementation +of the popularity transfer is out of scope for the data file format. Package IDs that do not appear as a key in this +file do not have their popularity transferred. + +The package ID keys and values are not guaranteed to have the original (author-intended) casing and should be treated +in a case insensitive manner. + +The order of the package ID keys and values is case insensitive ascending lexicographical order. + +The class for reading and writing this file to Blob Storage is [`PopularityTransferDataClient`](../src/NuGet.Services.AzureSearch/AuxiliaryFiles/PopularityTransferDataClient.cs). diff --git a/docs/Search-version-list-resource.md b/docs/Search-version-list-resource.md new file mode 100644 index 000000000..3370b93f8 --- /dev/null +++ b/docs/Search-version-list-resource.md @@ -0,0 +1,49 @@ +# Search version list resource + +**Subsystem: Search 🔎** + +The version list resource is bookkeeping used to update the search subsytem. Effectively, this resource is a mapping of package IDs to their versions. It is initially created by the [Db2AzureSearch](../src/NuGet.Jobs.Db2AzureSearch) tool and is kept up-to-date by the [Catalog2AzureSearch](../src/NuGet.Jobs.Catalog2AzureSearch) job. + +## Purpose + +The [search index](./Azure-Search-indexes.md#Search-index) stores up to 4 documents per package ID. These documents store the metadata for the latest listed version across these pivots: + +1. Prerelease - Includes packages that aren't stable +1. SemVer - Includes packages that require SemVer 2.0.0 support + +When packages are created, modified, or deleted we must decide which Azure Search documents must be updated. Furthermore, we need to decide which package version is the latest listed version across both prerelease and semver pivots. How do we do this? Using the search version list resource! + +## Content + +Version lists are stored in the Azure Blob Storage container for [search auxiliary files](Search-auxiliary-files.md), in the `version-lists` folder. There is a JSON blob for each package ID where the blob's name is the package ID, lower-cased. + +Say package `Foo.Bar` has four versions: + +* `1.0.0` - This version is unlisted and should be hidden from search results +* `2.0.0` - This version is listed +* `3.0.0` - This version is listed is requires SemVer 2.0.0 support +* `4.0.0-prerelease` This version is listed and is prerelease + +`Foo.Bar` would have a blob named `version-lists/foo.bar.json` with content: + +```json +{ + "VersionProperties": { + "1.0.0": {}, + "2.0.0": { + "Listed": true + }, + "3.0.0": { + "Listed": true, + "SemVer2": true + }, + "4.0.0": { + "Listed": true + }, + } +} +``` + +Notice that properties with `false` values are omitted. An unlisted version that does not require SemVer 2.0.0 does not have any properties. + +The order of versions within the `VersionProperties` object is undefined. \ No newline at end of file diff --git a/sign.thirdparty.props b/sign.thirdparty.props index 927874066..3df4ce762 100644 --- a/sign.thirdparty.props +++ b/sign.thirdparty.props @@ -5,9 +5,13 @@ + + + + @@ -23,7 +27,9 @@ + + diff --git a/src/Catalog/AggregateCursor.cs b/src/Catalog/AggregateCursor.cs new file mode 100644 index 000000000..6573f2fb7 --- /dev/null +++ b/src/Catalog/AggregateCursor.cs @@ -0,0 +1,35 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog +{ + /// + /// A that returns the least value for from a set of s. + /// + public class AggregateCursor : ReadCursor + { + public AggregateCursor(IEnumerable innerCursors) + { + if (innerCursors == null || !innerCursors.Any()) + { + throw new ArgumentException("Must supply at least one cursor!", nameof(innerCursors)); + } + + InnerCursors = innerCursors.ToList(); + } + + public IEnumerable InnerCursors { get; private set; } + + public override async Task LoadAsync(CancellationToken cancellationToken) + { + await Task.WhenAll(InnerCursors.Select(c => c.LoadAsync(cancellationToken))); + Value = InnerCursors.Min(c => c.Value); + } + } +} \ No newline at end of file diff --git a/src/Catalog/AppendOnlyCatalogItem.cs b/src/Catalog/AppendOnlyCatalogItem.cs new file mode 100644 index 000000000..3bd5a2c9b --- /dev/null +++ b/src/Catalog/AppendOnlyCatalogItem.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; + +namespace NuGet.Services.Metadata.Catalog +{ + public abstract class AppendOnlyCatalogItem : CatalogItem + { + public Uri GetBaseAddress() + { + return new Uri(BaseAddress, "data/" + MakeTimeStampPathComponent(TimeStamp)); + } + + protected virtual string GetItemIdentity() + { + return string.Empty; + } + + public string GetRelativeAddress() + { + return GetItemIdentity() + ".json"; + } + + public override Uri GetItemAddress() + { + return new Uri(GetBaseAddress(), GetRelativeAddress()); + } + + protected static string MakeTimeStampPathComponent(DateTime timeStamp) + { + return string.Format("{0:0000}.{1:00}.{2:00}.{3:00}.{4:00}.{5:00}/", timeStamp.Year, timeStamp.Month, timeStamp.Day, timeStamp.Hour, timeStamp.Minute, timeStamp.Second); + } + } +} diff --git a/src/Catalog/AppendOnlyCatalogWriter.cs b/src/Catalog/AppendOnlyCatalogWriter.cs new file mode 100644 index 000000000..39e5327a5 --- /dev/null +++ b/src/Catalog/AppendOnlyCatalogWriter.cs @@ -0,0 +1,144 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Persistence; +using VDS.RDF; + +namespace NuGet.Services.Metadata.Catalog +{ + public class AppendOnlyCatalogWriter : CatalogWriterBase + { + private readonly ITelemetryService _telemetryService; + private readonly bool _append; + private bool _first; + + public AppendOnlyCatalogWriter( + IStorage storage, + ITelemetryService telemetryService, + int maxPageSize = 1000, + bool append = true, + ICatalogGraphPersistence catalogGraphPersistence = null, + CatalogContext context = null) + : base(storage, catalogGraphPersistence, context) + { + _telemetryService = telemetryService; + _append = append; + _first = true; + MaxPageSize = maxPageSize; + } + + public int MaxPageSize + { + get; + private set; + } + + protected override Uri[] GetAdditionalRootType() + { + return new Uri[] { Schema.DataTypes.AppendOnlyCatalog, Schema.DataTypes.Permalink }; + } + + protected override async Task SaveRoot( + Guid commitId, + DateTime commitTimeStamp, + IDictionary pageEntries, + IGraph commitMetadata, + CancellationToken cancellationToken) + { + var stopwatch = Stopwatch.StartNew(); + await base.SaveRoot(commitId, commitTimeStamp, pageEntries, commitMetadata, cancellationToken); + _telemetryService.TrackCatalogIndexWriteDuration(stopwatch.Elapsed, RootUri); + } + + protected override async Task> SavePages(Guid commitId, DateTime commitTimeStamp, IDictionary itemEntries, CancellationToken cancellationToken) + { + IDictionary pageEntries; + if (_first && !_append) + { + pageEntries = new Dictionary(); + _first = false; + } + else + { + pageEntries = await LoadIndexResource(RootUri, cancellationToken); + } + + bool isExistingPage; + Uri pageUri = GetPageUri(pageEntries, itemEntries.Count, out isExistingPage); + + var items = new Dictionary(itemEntries); + + if (isExistingPage) + { + IDictionary existingItemEntries = await LoadIndexResource(pageUri, cancellationToken); + foreach (var entry in existingItemEntries) + { + items.Add(entry.Key, entry.Value); + } + } + + await SaveIndexResource(pageUri, Schema.DataTypes.CatalogPage, commitId, commitTimeStamp, items, RootUri, null, null, cancellationToken); + + pageEntries[pageUri.AbsoluteUri] = new CatalogItemSummary(Schema.DataTypes.CatalogPage, commitId, commitTimeStamp, items.Count); + + return pageEntries; + } + + private Uri GetPageUri(IDictionary currentPageEntries, int newItemCount, out bool isExistingPage) + { + Tuple latest = ExtractLatest(currentPageEntries); + int nextPageNumber = latest.Item1 + 1; + Uri latestUri = latest.Item2; + int latestCount = latest.Item3; + + isExistingPage = false; + + if (latestUri == null) + { + return CreatePageUri(Storage.BaseAddress, "page0"); + } + + if (latestCount + newItemCount > MaxPageSize) + { + return CreatePageUri(Storage.BaseAddress, string.Format("page{0}", nextPageNumber)); + } + + isExistingPage = true; + + return latestUri; + } + + private static Tuple ExtractLatest(IDictionary currentPageEntries) + { + int maxPageNumber = -1; + Uri latestUri = null; + int latestCount = 0; + + foreach (KeyValuePair entry in currentPageEntries) + { + int first = entry.Key.IndexOf("page") + 4; + int last = first; + while (last < entry.Key.Length && char.IsNumber(entry.Key, last)) + { + last++; + } + string s = entry.Key.Substring(first, last - first); + int pageNumber = int.Parse(s); + + if (pageNumber > maxPageNumber) + { + maxPageNumber = pageNumber; + latestUri = new Uri(entry.Key); + latestCount = entry.Value.Count.Value; + } + } + + return new Tuple(maxPageNumber, latestUri, latestCount); + } + } +} \ No newline at end of file diff --git a/src/Catalog/BatchProcessingException.cs b/src/Catalog/BatchProcessingException.cs new file mode 100644 index 000000000..cdc135cf1 --- /dev/null +++ b/src/Catalog/BatchProcessingException.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog +{ + public sealed class BatchProcessingException : Exception + { + public BatchProcessingException(Exception inner) + : base(Strings.BatchProcessingFailure, inner ?? throw new ArgumentNullException(nameof(inner))) + { + } + } +} \ No newline at end of file diff --git a/src/Catalog/CatalogCommit.cs b/src/Catalog/CatalogCommit.cs new file mode 100644 index 000000000..366eefd98 --- /dev/null +++ b/src/Catalog/CatalogCommit.cs @@ -0,0 +1,51 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Globalization; +using Newtonsoft.Json.Linq; + +namespace NuGet.Services.Metadata.Catalog +{ + /// + /// Represents a single catalog commit. + /// + public sealed class CatalogCommit : IComparable + { + private CatalogCommit(DateTime commitTimeStamp, Uri uri) + { + CommitTimeStamp = commitTimeStamp; + Uri = uri; + } + + public DateTime CommitTimeStamp { get; } + public Uri Uri { get; } + + public int CompareTo(object obj) + { + var other = obj as CatalogCommit; + + if (ReferenceEquals(other, null)) + { + throw new ArgumentException( + string.Format(CultureInfo.InvariantCulture, Strings.ArgumentMustBeInstanceOfType, nameof(CatalogCommit)), + nameof(obj)); + } + + return CommitTimeStamp.CompareTo(other.CommitTimeStamp); + } + + public static CatalogCommit Create(JObject commit) + { + if (commit == null) + { + throw new ArgumentNullException(nameof(commit)); + } + + var commitTimeStamp = Utils.Deserialize(commit, "commitTimeStamp"); + var uri = Utils.Deserialize(commit, "@id"); + + return new CatalogCommit(commitTimeStamp, uri); + } + } +} \ No newline at end of file diff --git a/src/Catalog/CatalogCommitItem.cs b/src/Catalog/CatalogCommitItem.cs new file mode 100644 index 000000000..4e0a1f505 --- /dev/null +++ b/src/Catalog/CatalogCommitItem.cs @@ -0,0 +1,126 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using Newtonsoft.Json.Linq; +using NuGet.Packaging.Core; +using NuGet.Versioning; + +namespace NuGet.Services.Metadata.Catalog +{ + /// + /// Represents a single item in a catalog commit. + /// + [DebuggerDisplay("{DebuggerDisplay,nq}")] + public sealed class CatalogCommitItem : IComparable + { + private string DebuggerDisplay => + $"Catalog item {PackageIdentity?.Id} {PackageIdentity?.Version.ToNormalizedString()}" + + $"{(IsPackageDetails ? " (" + nameof(Schema.DataTypes.PackageDetails) + ")" : string.Empty)}" + + $"{(IsPackageDelete ? " (" + nameof(Schema.DataTypes.PackageDelete) + ")" : string.Empty)}"; + + private const string _typeKeyword = "@type"; + + public CatalogCommitItem( + Uri uri, + string commitId, + DateTime commitTimeStamp, + IReadOnlyList types, + IReadOnlyList typeUris, + PackageIdentity packageIdentity) + { + Uri = uri; + CommitId = commitId; + CommitTimeStamp = commitTimeStamp; + PackageIdentity = packageIdentity; + Types = types; + TypeUris = typeUris; + + IsPackageDetails = HasTypeUri(Schema.DataTypes.PackageDetails); + IsPackageDelete = HasTypeUri(Schema.DataTypes.PackageDelete); + } + + public Uri Uri { get; } + public DateTime CommitTimeStamp { get; } + public string CommitId { get; } + public PackageIdentity PackageIdentity { get; } + public IReadOnlyList Types { get; } + public IReadOnlyList TypeUris { get; } + + public bool IsPackageDetails { get; } + public bool IsPackageDelete { get; } + + public bool HasTypeUri(Uri typeUri) + { + return TypeUris.Any(x => x.IsAbsoluteUri && x.AbsoluteUri == typeUri.AbsoluteUri); + } + + public int CompareTo(object obj) + { + var other = obj as CatalogCommitItem; + + if (ReferenceEquals(other, null)) + { + throw new ArgumentException( + string.Format(CultureInfo.InvariantCulture, Strings.ArgumentMustBeInstanceOfType, nameof(CatalogCommitItem)), + nameof(obj)); + } + + return CommitTimeStamp.CompareTo(other.CommitTimeStamp); + } + + public static CatalogCommitItem Create(JObject context, JObject commitItem) + { + if (context == null) + { + throw new ArgumentNullException(nameof(context)); + } + + if (commitItem == null) + { + throw new ArgumentNullException(nameof(commitItem)); + } + + var commitTimeStamp = Utils.Deserialize(commitItem, "commitTimeStamp"); + var commitId = Utils.Deserialize(commitItem, "commitId"); + var idUri = Utils.Deserialize(commitItem, "@id"); + var packageId = Utils.Deserialize(commitItem, "nuget:id"); + var packageVersion = Utils.Deserialize(commitItem, "nuget:version"); + var packageIdentity = new PackageIdentity(packageId, new NuGetVersion(packageVersion)); + var types = GetTypes(commitItem).ToArray(); + + if (!types.Any()) + { + throw new ArgumentException( + string.Format(CultureInfo.InvariantCulture, Strings.NonEmptyPropertyValueRequired, _typeKeyword), + nameof(commitItem)); + } + + var typeUris = types.Select(type => Utils.Expand(context, type)).ToArray(); + + return new CatalogCommitItem(idUri, commitId, commitTimeStamp, types, typeUris, packageIdentity); + } + + private static IEnumerable GetTypes(JObject commitItem) + { + if (commitItem.TryGetValue(_typeKeyword, out var value)) + { + if (value is JArray) + { + foreach (JToken typeToken in ((JArray)value).Values()) + { + yield return typeToken.ToString(); + } + } + else + { + yield return value.ToString(); + } + } + } + } +} \ No newline at end of file diff --git a/src/Catalog/CatalogCommitItemBatch.cs b/src/Catalog/CatalogCommitItemBatch.cs new file mode 100644 index 000000000..3fc3a28d3 --- /dev/null +++ b/src/Catalog/CatalogCommitItemBatch.cs @@ -0,0 +1,49 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace NuGet.Services.Metadata.Catalog +{ + /// + /// Represents a group of . + /// Items may span multiple commits but are grouped on common criteria (e.g.: lower-cased package ID). + /// + public sealed class CatalogCommitItemBatch + { + /// + /// Initializes a instance. + /// + /// An enumerable of . Items may span multiple commits. + /// A unique key for all items in a batch. This is used for parallelization and may be + /// null if parallelization is not used. + /// The commit timestamp to use for this batch. If null, the minimum of + /// all item commit timestamps will be used. + /// Thrown if is either null or empty. + public CatalogCommitItemBatch( + IEnumerable items, + string key = null, + DateTime? commitTimestamp = null) + { + if (items == null || !items.Any()) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(items)); + } + + var list = items.ToList(); + + CommitTimeStamp = commitTimestamp ?? list.Min(item => item.CommitTimeStamp); + Key = key; + + list.Sort(); + + Items = list; + } + + public DateTime CommitTimeStamp { get; } + public IReadOnlyList Items { get; } + public string Key { get; } + } +} \ No newline at end of file diff --git a/src/Catalog/CatalogCommitItemBatchTask.cs b/src/Catalog/CatalogCommitItemBatchTask.cs new file mode 100644 index 000000000..3d8e58401 --- /dev/null +++ b/src/Catalog/CatalogCommitItemBatchTask.cs @@ -0,0 +1,68 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog +{ + /// + /// Represents an asynchrononous task associated with catalog changes for a specific + /// and potentially spanning multiple commits. + /// + public sealed class CatalogCommitItemBatchTask : IEquatable + { + /// + /// Initializes a instance. + /// + /// A . + /// A tracking completion of + /// processing. + /// Thrown if is null. + /// Thrown if is null. + /// Thrown if is null. + public CatalogCommitItemBatchTask(CatalogCommitItemBatch batch, Task task) + { + if (batch == null) + { + throw new ArgumentNullException(nameof(batch)); + } + + if (batch.Key == null) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNull, $"{nameof(batch)}.{nameof(batch.Key)}"); + } + + if (task == null) + { + throw new ArgumentNullException(nameof(task)); + } + + Batch = batch; + Task = task; + } + + public CatalogCommitItemBatch Batch { get; } + public Task Task { get; } + + public override int GetHashCode() + { + return Batch.Key.GetHashCode(); + } + + public override bool Equals(object obj) + { + return Equals(obj as CatalogCommitItemBatchTask); + } + + public bool Equals(CatalogCommitItemBatchTask other) + { + if (ReferenceEquals(other, null)) + { + return false; + } + + return string.Equals(Batch.Key, other.Batch.Key); + } + } +} \ No newline at end of file diff --git a/src/Catalog/CatalogCommitUtilities.cs b/src/Catalog/CatalogCommitUtilities.cs new file mode 100644 index 000000000..1597ec8ff --- /dev/null +++ b/src/Catalog/CatalogCommitUtilities.cs @@ -0,0 +1,323 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json.Linq; +using ExceptionUtilities = NuGet.Common.ExceptionUtilities; + +namespace NuGet.Services.Metadata.Catalog +{ + public static class CatalogCommitUtilities + { + private static readonly EventId _eventId = new EventId(id: 0); + + /// + /// Creates an enumerable of instances. + /// + /// + /// A instance contains only the latest commit for each package identity. + /// + /// An enumerable of . + /// A function that returns a key for a . + /// An enumerable of with no ordering guarantee. + /// Thrown if is null. + /// Thrown if is null. + public static IEnumerable CreateCommitItemBatches( + IEnumerable catalogItems, + GetCatalogCommitItemKey getCatalogCommitItemKey) + { + if (catalogItems == null) + { + throw new ArgumentNullException(nameof(catalogItems)); + } + + if (getCatalogCommitItemKey == null) + { + throw new ArgumentNullException(nameof(getCatalogCommitItemKey)); + } + + var catalogItemsGroups = catalogItems + .GroupBy(catalogItem => getCatalogCommitItemKey(catalogItem)); + + var batches = new List(); + + foreach (var catalogItemsGroup in catalogItemsGroups) + { + var catalogItemsWithOnlyLatestCommitForEachPackageIdentity = catalogItemsGroup + .GroupBy(commitItem => new + { + PackageId = commitItem.PackageIdentity.Id.ToLowerInvariant(), + PackageVersion = commitItem.PackageIdentity.Version.ToNormalizedString().ToLowerInvariant() + }) + .Select(group => group.OrderBy(item => item.CommitTimeStamp).Last()) + .ToArray(); + var minCommitTimeStamp = catalogItemsWithOnlyLatestCommitForEachPackageIdentity + .Select(catalogItem => catalogItem.CommitTimeStamp) + .Min(); + + batches.Add( + new CatalogCommitItemBatch( + catalogItemsWithOnlyLatestCommitForEachPackageIdentity, + catalogItemsGroup.Key)); + } + + // Assert only after skipping older commits for each package identity to reduce the likelihood + // of unnecessary failures. + AssertNotMoreThanOneCommitIdPerCommitTimeStamp(batches, nameof(catalogItems)); + + return batches; + } + + public static void StartProcessingBatchesIfNoFailures( + CollectorHttpClient client, + JToken context, + List unprocessedBatches, + List processingBatches, + int maxConcurrentBatches, + ProcessCommitItemBatchAsync processCommitItemBatchAsync, + CancellationToken cancellationToken) + { + if (client == null) + { + throw new ArgumentNullException(nameof(client)); + } + + if (context == null) + { + throw new ArgumentNullException(nameof(context)); + } + + if (unprocessedBatches == null) + { + throw new ArgumentNullException(nameof(unprocessedBatches)); + } + + if (processingBatches == null) + { + throw new ArgumentNullException(nameof(processingBatches)); + } + + if (maxConcurrentBatches < 1) + { + throw new ArgumentOutOfRangeException( + nameof(maxConcurrentBatches), + maxConcurrentBatches, + string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue)); + } + + if (processCommitItemBatchAsync == null) + { + throw new ArgumentNullException(nameof(processCommitItemBatchAsync)); + } + + var hasAnyBatchFailed = processingBatches.Any(batch => batch.Task.IsFaulted || batch.Task.IsCanceled); + + if (hasAnyBatchFailed) + { + return; + } + + var batchesToEnqueue = Math.Min( + maxConcurrentBatches - processingBatches.Count(batch => !batch.Task.IsCompleted), + unprocessedBatches.Count); + + for (var i = 0; i < batchesToEnqueue; ++i) + { + var batch = unprocessedBatches[0]; + + unprocessedBatches.RemoveAt(0); + + var task = processCommitItemBatchAsync( + client, + context, + batch.Key, + batch, + lastBatch: null, + cancellationToken: cancellationToken); + var batchTask = new CatalogCommitItemBatchTask(batch, task); + + processingBatches.Add(batchTask); + } + } + + internal static async Task ProcessCatalogCommitsAsync( + CollectorHttpClient client, + ReadWriteCursor front, + ReadCursor back, + FetchCatalogCommitsAsync fetchCatalogCommitsAsync, + CreateCommitItemBatchesAsync createCommitItemBatchesAsync, + ProcessCommitItemBatchAsync processCommitItemBatchAsync, + int maxConcurrentBatches, + ILogger logger, + CancellationToken cancellationToken) + { + var rootItems = await fetchCatalogCommitsAsync(client, front, back, cancellationToken); + + var hasAnyBatchFailed = false; + var hasAnyBatchBeenProcessed = false; + + foreach (CatalogCommit rootItem in rootItems) + { + JObject page = await client.GetJObjectAsync(rootItem.Uri, cancellationToken); + var context = (JObject)page["@context"]; + CatalogCommitItemBatch[] batches = await CreateBatchesForAllAvailableItemsInPageAsync( + front, + back, + page, + context, + createCommitItemBatchesAsync); + + if (!batches.Any()) + { + continue; + } + + hasAnyBatchBeenProcessed = true; + + DateTime maxCommitTimeStamp = GetMaxCommitTimeStamp(batches); + var unprocessedBatches = batches.ToList(); + var processingBatches = new List(); + var exceptions = new List(); + + StartProcessingBatchesIfNoFailures( + client, + context, + unprocessedBatches, + processingBatches, + maxConcurrentBatches, + processCommitItemBatchAsync, + cancellationToken); + + while (processingBatches.Any()) + { + var activeTasks = processingBatches.Where(batch => !batch.Task.IsCompleted) + .Select(batch => batch.Task) + .DefaultIfEmpty(Task.CompletedTask); + + await Task.WhenAny(activeTasks); + + for (var i = 0; i < processingBatches.Count; ++i) + { + var batch = processingBatches[i]; + + if (batch.Task.IsFaulted || batch.Task.IsCanceled) + { + hasAnyBatchFailed = true; + + if (batch.Task.Exception != null) + { + var exception = ExceptionUtilities.Unwrap(batch.Task.Exception); + + exceptions.Add(exception); + } + } + + if (batch.Task.IsCompleted) + { + processingBatches.RemoveAt(i); + --i; + } + } + + if (!hasAnyBatchFailed) + { + StartProcessingBatchesIfNoFailures( + client, + context, + unprocessedBatches, + processingBatches, + maxConcurrentBatches, + processCommitItemBatchAsync, + cancellationToken); + } + } + + if (hasAnyBatchFailed) + { + foreach (var exception in exceptions) + { + logger.LogError(_eventId, exception, Strings.BatchProcessingFailure); + } + + var innerException = exceptions.Count == 1 ? exceptions.Single() : new AggregateException(exceptions); + + throw new BatchProcessingException(innerException); + } + + front.Value = maxCommitTimeStamp; + + await front.SaveAsync(cancellationToken); + + Trace.TraceInformation($"{nameof(CatalogCommitUtilities)}.{nameof(ProcessCatalogCommitsAsync)} " + + $"{nameof(front)}.{nameof(front.Value)} saved since timestamp changed from previous: {{0}}", front); + } + + return hasAnyBatchBeenProcessed; + } + + public static string GetPackageIdKey(CatalogCommitItem item) + { + if (item == null) + { + throw new ArgumentNullException(nameof(item)); + } + + return item.PackageIdentity.Id.ToLowerInvariant(); + } + + private static async Task CreateBatchesForAllAvailableItemsInPageAsync( + ReadWriteCursor front, + ReadCursor back, + JObject page, + JObject context, + CreateCommitItemBatchesAsync createCommitItemBatchesAsync) + { + IEnumerable commitItems = page["items"] + .Select(item => CatalogCommitItem.Create(context, (JObject)item)) + .Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value); + + IEnumerable batches = await createCommitItemBatchesAsync(commitItems); + + return batches + .OrderBy(batch => batch.CommitTimeStamp) + .ToArray(); + } + + private static DateTime GetMaxCommitTimeStamp(CatalogCommitItemBatch[] batches) + { + return batches.SelectMany(batch => batch.Items) + .Select(item => item.CommitTimeStamp) + .Max(); + } + + private static void AssertNotMoreThanOneCommitIdPerCommitTimeStamp( + IEnumerable batches, + string parameterName) + { + var commitsWithSameTimeStampButDifferentCommitIds = batches + .SelectMany(batch => batch.Items) + .GroupBy(commitItem => commitItem.CommitTimeStamp) + .Where(group => group.Select(item => item.CommitId).Distinct().Count() > 1); + + if (commitsWithSameTimeStampButDifferentCommitIds.Any()) + { + var commits = commitsWithSameTimeStampButDifferentCommitIds.SelectMany(group => group) + .Select(commit => $"{{ CommitId = {commit.CommitId}, CommitTimeStamp = {commit.CommitTimeStamp.ToString("O")} }}"); + + throw new ArgumentException( + string.Format( + CultureInfo.InvariantCulture, + Strings.MultipleCommitIdsForSameCommitTimeStamp, + string.Join(", ", commits)), + parameterName); + } + } + } +} \ No newline at end of file diff --git a/src/Catalog/CatalogContext.cs b/src/Catalog/CatalogContext.cs new file mode 100644 index 000000000..7f764c99f --- /dev/null +++ b/src/Catalog/CatalogContext.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Concurrent; +using System.IO; + +namespace NuGet.Services.Metadata.Catalog +{ + public class CatalogContext + { + ConcurrentDictionary _jsonLdContext; + + public CatalogContext() + { + _jsonLdContext = new ConcurrentDictionary(); + } + + public JObject GetJsonLdContext(string name, Uri type) + { + return _jsonLdContext.GetOrAdd(name + "#" + type.ToString(), (key) => + { + using (JsonReader jsonReader = new JsonTextReader(new StreamReader(Utils.GetResourceStream(name)))) + { + JObject obj = JObject.Load(jsonReader); + obj["@type"] = type.ToString(); + return obj; + } + }); + } + } +} diff --git a/src/Catalog/CatalogIndexEntry.cs b/src/Catalog/CatalogIndexEntry.cs new file mode 100644 index 000000000..48112b8dc --- /dev/null +++ b/src/Catalog/CatalogIndexEntry.cs @@ -0,0 +1,143 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Newtonsoft.Json; +using NuGet.Packaging.Core; +using NuGet.Versioning; + +namespace NuGet.Services.Metadata.Catalog +{ + public sealed class CatalogIndexEntry : IComparable + { + [JsonConstructor] + private CatalogIndexEntry() + { + Types = Enumerable.Empty(); + } + + public CatalogIndexEntry( + Uri uri, + string type, + string commitId, + DateTime commitTs, + PackageIdentity packageIdentity) + { + if (string.IsNullOrWhiteSpace(type)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullEmptyOrWhitespace, nameof(type)); + } + + Initialize(uri, new[] { type }, commitId, commitTs, packageIdentity); + } + + public CatalogIndexEntry( + Uri uri, + IReadOnlyList types, + string commitId, + DateTime commitTs, + PackageIdentity packageIdentity) + { + Initialize(uri, types, commitId, commitTs, packageIdentity); + } + + [JsonProperty("@id")] + [JsonRequired] + public Uri Uri { get; private set; } + + [JsonProperty("@type")] + [JsonRequired] + [JsonConverter(typeof(CatalogTypeConverter))] + public IEnumerable Types { get; private set; } + + [JsonProperty("commitId")] + [JsonRequired] + public string CommitId { get; private set; } + + [JsonProperty("commitTimeStamp")] + [JsonRequired] + public DateTime CommitTimeStamp { get; private set; } + + [JsonProperty("nuget:id")] + [JsonRequired] + public string Id { get; private set; } + + [JsonProperty("nuget:version")] + [JsonRequired] + public NuGetVersion Version { get; private set; } + + [JsonIgnore] + public bool IsDelete + { + get + { + return Types.Any(type => type == "nuget:PackageDelete"); + } + } + + public int CompareTo(CatalogIndexEntry other) + { + if (other == null) + { + throw new ArgumentNullException(nameof(other)); + } + + return CommitTimeStamp.CompareTo(other.CommitTimeStamp); + } + + public static CatalogIndexEntry Create(CatalogCommitItem commitItem) + { + if (commitItem == null) + { + throw new ArgumentNullException(nameof(commitItem)); + } + + return new CatalogIndexEntry( + commitItem.Uri, + commitItem.Types, + commitItem.CommitId, + commitItem.CommitTimeStamp, + commitItem.PackageIdentity); + } + + private void Initialize( + Uri uri, + IReadOnlyList types, + string commitId, + DateTime commitTs, + PackageIdentity packageIdentity) + { + Uri = uri ?? throw new ArgumentNullException(nameof(uri)); + + if (types == null || !types.Any()) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(types)); + } + + if (types.Any(type => string.IsNullOrWhiteSpace(type))) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullEmptyOrWhitespace, nameof(types)); + } + + Types = types; + + if (string.IsNullOrWhiteSpace(commitId)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(commitId)); + } + + CommitId = commitId; + CommitTimeStamp = commitTs; + + if (packageIdentity == null) + { + throw new ArgumentNullException(nameof(packageIdentity)); + } + + Id = packageIdentity.Id; + Version = packageIdentity.Version; + } + } +} \ No newline at end of file diff --git a/src/Catalog/CatalogIndexReader.cs b/src/Catalog/CatalogIndexReader.cs new file mode 100644 index 000000000..e33b5a4a9 --- /dev/null +++ b/src/Catalog/CatalogIndexReader.cs @@ -0,0 +1,105 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NuGet.Packaging.Core; +using NuGet.Versioning; + +namespace NuGet.Services.Metadata.Catalog +{ + public class CatalogIndexReader + { + private readonly Uri _indexUri; + private readonly CollectorHttpClient _httpClient; + private readonly ITelemetryService _telemetryService; + private JObject _context; + + public CatalogIndexReader(Uri indexUri, CollectorHttpClient httpClient, ITelemetryService telemetryService) + { + _indexUri = indexUri; + _httpClient = httpClient; + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + } + + public async Task> GetEntries() + { + var stopwatch = Stopwatch.StartNew(); + JObject index = await _httpClient.GetJObjectAsync(_indexUri); + _telemetryService.TrackCatalogIndexReadDuration(stopwatch.Elapsed, _indexUri); + + // save the context used on the index + JToken context = null; + if (index.TryGetValue("@context", out context)) + { + _context = context as JObject; + } + + List> pages = new List>(); + + foreach (var item in index["items"]) + { + pages.Add(new Tuple(DateTime.Parse(item["commitTimeStamp"].ToString()), new Uri(item["@id"].ToString()))); + } + + return await GetEntriesAsync(pages.Select(p => p.Item2)); + } + + private async Task> GetEntriesAsync(IEnumerable pageUris) + { + var pageUriBag = new ConcurrentBag(pageUris); + var entries = new ConcurrentBag(); + var interner = new StringInterner(); + + var tasks = Enumerable + .Range(0, ServicePointManager.DefaultConnectionLimit) + .Select(i => ProcessPageUris(pageUriBag, entries, interner)) + .ToList(); + + await Task.WhenAll(tasks); + + return entries; + } + + private async Task ProcessPageUris(ConcurrentBag pageUriBag, ConcurrentBag entries, StringInterner interner) + { + await Task.Yield(); + Uri pageUri; + while (pageUriBag.TryTake(out pageUri)) + { + var json = await _httpClient.GetJObjectAsync(pageUri); + + foreach (var item in json["items"]) + { + // This string is unique. + var id = item["@id"].ToString(); + + // These strings should be shared. + var type = interner.Intern(item["@type"].ToString()); + var commitId = interner.Intern(item["commitId"].ToString()); + var nugetId = interner.Intern(item["nuget:id"].ToString()); + var nugetVersion = interner.Intern(item["nuget:version"].ToString()); + var packageIdentity = new PackageIdentity(nugetId, NuGetVersion.Parse(nugetVersion)); + + // No string is directly operated on here. + var commitTimeStamp = item["commitTimeStamp"].ToObject(); + + var entry = new CatalogIndexEntry( + new Uri(id), + type, + commitId, + commitTimeStamp, + packageIdentity); + + entries.Add(entry); + } + } + } + } +} \ No newline at end of file diff --git a/src/Catalog/CatalogItem.cs b/src/Catalog/CatalogItem.cs new file mode 100644 index 000000000..51a9ddad4 --- /dev/null +++ b/src/Catalog/CatalogItem.cs @@ -0,0 +1,40 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Services.Metadata.Catalog.Persistence; +using VDS.RDF; + +namespace NuGet.Services.Metadata.Catalog +{ + public abstract class CatalogItem + { + public DateTime TimeStamp { get; set; } + + public Guid CommitId { get; set; } + + public Uri BaseAddress { get; set; } + + public abstract Uri GetItemType(); + + public abstract Uri GetItemAddress(); + + public virtual StorageContent CreateContent(CatalogContext context) + { + return null; + } + + public virtual IGraph CreatePageContent(CatalogContext context) + { + return null; + } + + /// + /// Create the core graph used in CreateContent(context) + /// + public virtual IGraph CreateContentGraph(CatalogContext context) + { + return null; + } + } +} diff --git a/src/Catalog/CatalogItemSummary.cs b/src/Catalog/CatalogItemSummary.cs new file mode 100644 index 000000000..3dbbef9f7 --- /dev/null +++ b/src/Catalog/CatalogItemSummary.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; +using VDS.RDF; + +namespace NuGet.Services.Metadata.Catalog +{ + public class CatalogItemSummary + { + public CatalogItemSummary(Uri type, Guid commitId, DateTime commitTimeStamp, int? count = null, IGraph content = null) + { + Type = type; + CommitId = commitId; + CommitTimeStamp = commitTimeStamp; + Count = count; + Content = content; + } + + public Uri Type { get; private set; } + public Guid CommitId { get; private set; } + public DateTime CommitTimeStamp { get; private set; } + public int? Count { get; private set; } + public IGraph Content { get; private set; } + } +} diff --git a/src/Catalog/CatalogTypeConverter.cs b/src/Catalog/CatalogTypeConverter.cs new file mode 100644 index 000000000..c2e433924 --- /dev/null +++ b/src/Catalog/CatalogTypeConverter.cs @@ -0,0 +1,53 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Newtonsoft.Json; + +namespace NuGet.Services.Metadata.Catalog +{ + public class CatalogTypeConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(IEnumerable); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + object type = null; + if (reader.TokenType == JsonToken.StartArray) + { + // If the type is stored in an array, use the last value and then discard the rest. + do + { + reader.Read(); + if (reader.TokenType == JsonToken.String) + { + type = reader.Value; + } + } while (reader.TokenType != JsonToken.EndArray); + } + else + { + type = reader.Value; + } + + if (type == null) + { + throw new InvalidDataException("Failed to parse the type of a catalog entry!"); + } + + return new string[] { type.ToString() }; + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + var types = value as IEnumerable; + serializer.Serialize(writer, types.First()); + } + } +} \ No newline at end of file diff --git a/src/Catalog/CatalogWriterBase.cs b/src/Catalog/CatalogWriterBase.cs new file mode 100644 index 000000000..a5a8bead9 --- /dev/null +++ b/src/Catalog/CatalogWriterBase.cs @@ -0,0 +1,381 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NuGet.Services.Metadata.Catalog.Persistence; +using VDS.RDF; + +namespace NuGet.Services.Metadata.Catalog +{ + public abstract class CatalogWriterBase : IDisposable + { + protected List _batch; + protected bool _open; + + public CatalogWriterBase(IStorage storage, ICatalogGraphPersistence graphPersistence = null, CatalogContext context = null) + { + Options.InternUris = false; + + Storage = storage; + GraphPersistence = graphPersistence; + Context = context ?? new CatalogContext(); + + _batch = new List(); + _open = true; + + RootUri = Storage.ResolveUri("index.json"); + } + + public void Dispose() + { + _batch.Clear(); + _open = false; + } + + public IStorage Storage { get; private set; } + + public ICatalogGraphPersistence GraphPersistence { get; private set; } + + public Uri RootUri { get; private set; } + + public CatalogContext Context { get; private set; } + + public int Count { get { return _batch.Count; } } + + public void Add(CatalogItem item) + { + if (!_open) + { + throw new ObjectDisposedException(GetType().Name); + } + + _batch.Add(item); + } + public Task> Commit(IGraph commitMetadata, CancellationToken cancellationToken) + { + return Commit(DateTime.UtcNow, commitMetadata, cancellationToken); + } + + public virtual async Task> Commit(DateTime commitTimeStamp, IGraph commitMetadata, CancellationToken cancellationToken) + { + if (!_open) + { + throw new ObjectDisposedException(GetType().Name); + } + + if (_batch.Count == 0) + { + return Enumerable.Empty(); + } + + // the commitId is only used for tracing and trouble shooting + + Guid commitId = Guid.NewGuid(); + + // save items + + IDictionary newItemEntries = await SaveItems(commitId, commitTimeStamp, cancellationToken); + + // save index pages - this is abstract as the derived class determines the index pagination + + IDictionary pageEntries = await SavePages(commitId, commitTimeStamp, newItemEntries, cancellationToken); + + // save index root + + await SaveRoot(commitId, commitTimeStamp, pageEntries, commitMetadata, cancellationToken); + + _batch.Clear(); + + return newItemEntries.Keys.Select(s => new Uri(s)); + } + + private async Task> SaveItems(Guid commitId, DateTime commitTimeStamp, CancellationToken cancellationToken) + { + ConcurrentDictionary pageItems = new ConcurrentDictionary(); + + int batchIndex = 0; + + var saveTasks = new List(); + foreach (CatalogItem item in _batch) + { + ResourceSaveOperation saveOperationForItem = null; + + try + { + item.TimeStamp = commitTimeStamp; + item.CommitId = commitId; + item.BaseAddress = Storage.BaseAddress; + + saveOperationForItem = CreateSaveOperationForItem(Storage, Context, item, cancellationToken); + if (saveOperationForItem.SaveTask != null) + { + saveTasks.Add(saveOperationForItem.SaveTask); + } + + IGraph pageContent = item.CreatePageContent(Context); + + if (!pageItems.TryAdd(saveOperationForItem.ResourceUri.AbsoluteUri, new CatalogItemSummary(item.GetItemType(), commitId, commitTimeStamp, null, pageContent))) + { + throw new Exception("Duplicate page: " + saveOperationForItem.ResourceUri.AbsoluteUri); + } + + batchIndex++; + } + catch (Exception e) + { + string msg = (saveOperationForItem == null || saveOperationForItem.ResourceUri == null) + ? string.Format("batch index: {0}", batchIndex) + : string.Format("batch index: {0} resourceUri: {1}", batchIndex, saveOperationForItem.ResourceUri); + + throw new Exception(msg, e); + } + } + + await Task.WhenAll(saveTasks); + + return pageItems; + } + + protected virtual ResourceSaveOperation CreateSaveOperationForItem(IStorage storage, CatalogContext context, CatalogItem item, CancellationToken cancellationToken) + { + // This method decides what to do with the item. + // Standard method of operation: if content == null, don't do a thing. Else, write content. + + var content = item.CreateContent(Context); // note: always do this first + var resourceUri = item.GetItemAddress(); + + var operation = new ResourceSaveOperation(); + operation.ResourceUri = resourceUri; + + if (content != null) + { + operation.SaveTask = storage.SaveAsync(resourceUri, content, cancellationToken); + } + + return operation; + } + + protected virtual async Task SaveRoot(Guid commitId, DateTime commitTimeStamp, IDictionary pageEntries, IGraph commitMetadata, CancellationToken cancellationToken) + { + await SaveIndexResource(RootUri, Schema.DataTypes.CatalogRoot, commitId, commitTimeStamp, pageEntries, null, commitMetadata, GetAdditionalRootType(), cancellationToken); + } + + protected virtual Uri[] GetAdditionalRootType() + { + return null; + } + + protected abstract Task> SavePages(Guid commitId, DateTime commitTimeStamp, IDictionary itemEntries, CancellationToken cancellationToken); + + protected virtual StorageContent CreateIndexContent(IGraph graph, Uri type) + { + JObject frame = Context.GetJsonLdContext("context.Container.json", type); + return new JTokenStorageContent(Utils.CreateJson(graph, frame), "application/json", "no-store"); + } + + protected async Task SaveIndexResource(Uri resourceUri, Uri typeUri, Guid commitId, DateTime commitTimeStamp, IDictionary entries, Uri parent, IGraph extra, Uri[] additionalResourceTypes, CancellationToken cancellationToken) + { + IGraph graph = new Graph(); + + INode resourceNode = graph.CreateUriNode(resourceUri); + INode itemPredicate = graph.CreateUriNode(Schema.Predicates.CatalogItem); + INode typePredicate = graph.CreateUriNode(Schema.Predicates.Type); + INode timeStampPredicate = graph.CreateUriNode(Schema.Predicates.CatalogTimeStamp); + INode commitIdPredicate = graph.CreateUriNode(Schema.Predicates.CatalogCommitId); + INode countPredicate = graph.CreateUriNode(Schema.Predicates.CatalogCount); + + graph.Assert(resourceNode, typePredicate, graph.CreateUriNode(typeUri)); + graph.Assert(resourceNode, commitIdPredicate, graph.CreateLiteralNode(commitId.ToString())); + graph.Assert(resourceNode, timeStampPredicate, graph.CreateLiteralNode(commitTimeStamp.ToString("O"), Schema.DataTypes.DateTime)); + graph.Assert(resourceNode, countPredicate, graph.CreateLiteralNode(entries.Count.ToString(), Schema.DataTypes.Integer)); + + foreach (KeyValuePair itemEntry in entries) + { + INode itemNode = graph.CreateUriNode(new Uri(itemEntry.Key)); + + graph.Assert(resourceNode, itemPredicate, itemNode); + graph.Assert(itemNode, typePredicate, graph.CreateUriNode(itemEntry.Value.Type)); + graph.Assert(itemNode, commitIdPredicate, graph.CreateLiteralNode(itemEntry.Value.CommitId.ToString())); + graph.Assert(itemNode, timeStampPredicate, graph.CreateLiteralNode(itemEntry.Value.CommitTimeStamp.ToString("O"), Schema.DataTypes.DateTime)); + + if (itemEntry.Value.Count != null) + { + graph.Assert(itemNode, countPredicate, graph.CreateLiteralNode(itemEntry.Value.Count.ToString(), Schema.DataTypes.Integer)); + } + + if (itemEntry.Value.Content != null) + { + graph.Merge(itemEntry.Value.Content, true); + } + } + + if (parent != null) + { + graph.Assert(resourceNode, graph.CreateUriNode(Schema.Predicates.CatalogParent), graph.CreateUriNode(parent)); + } + + if (extra != null) + { + graph.Merge(extra, true); + } + + if (additionalResourceTypes != null) + { + foreach (Uri resourceType in additionalResourceTypes) + { + graph.Assert(resourceNode, typePredicate, graph.CreateUriNode(resourceType)); + } + } + + await SaveGraph(resourceUri, graph, typeUri, cancellationToken); + } + + protected async Task> LoadIndexResource(Uri resourceUri, CancellationToken cancellationToken) + { + IDictionary entries = new Dictionary(); + + IGraph graph = await LoadGraph(resourceUri, cancellationToken); + + if (graph == null) + { + return entries; + } + + INode typePredicate = graph.CreateUriNode(Schema.Predicates.Type); + INode itemPredicate = graph.CreateUriNode(Schema.Predicates.CatalogItem); + INode timeStampPredicate = graph.CreateUriNode(Schema.Predicates.CatalogTimeStamp); + INode commitIdPredicate = graph.CreateUriNode(Schema.Predicates.CatalogCommitId); + INode countPredicate = graph.CreateUriNode(Schema.Predicates.CatalogCount); + + CheckScheme(resourceUri, graph); + + INode resourceNode = graph.CreateUriNode(resourceUri); + + foreach (IUriNode itemNode in graph.GetTriplesWithSubjectPredicate(resourceNode, itemPredicate).Select((t) => t.Object)) + { + Triple typeTriple = graph.GetTriplesWithSubjectPredicate(itemNode, typePredicate).First(); + Uri type = ((IUriNode)typeTriple.Object).Uri; + + Triple commitIdTriple = graph.GetTriplesWithSubjectPredicate(itemNode, commitIdPredicate).First(); + Guid commitId = Guid.Parse(((ILiteralNode)commitIdTriple.Object).Value); + + Triple timeStampTriple = graph.GetTriplesWithSubjectPredicate(itemNode, timeStampPredicate).First(); + DateTime timeStamp = DateTime.Parse(((ILiteralNode)timeStampTriple.Object).Value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + + Triple countTriple = graph.GetTriplesWithSubjectPredicate(itemNode, countPredicate).FirstOrDefault(); + int? count = (countTriple != null) ? int.Parse(((ILiteralNode)countTriple.Object).Value) : (int?)null; + + IGraph itemContent = null; + INode itemContentSubjectNode = null; + foreach (Triple itemContentTriple in graph.GetTriplesWithSubject(itemNode)) + { + if (itemContentTriple.Predicate.Equals(typePredicate)) + { + continue; + } + if (itemContentTriple.Predicate.Equals(timeStampPredicate)) + { + continue; + } + if (itemContentTriple.Predicate.Equals(commitIdPredicate)) + { + continue; + } + if (itemContentTriple.Predicate.Equals(countPredicate)) + { + continue; + } + if (itemContentTriple.Predicate.Equals(itemPredicate)) + { + continue; + } + + if (itemContent == null) + { + itemContent = new Graph(); + itemContentSubjectNode = itemContentTriple.Subject.CopyNode(itemContent, false); + } + + INode itemContentPredicateNode = itemContentTriple.Predicate.CopyNode(itemContent, false); + INode itemContentObjectNode = itemContentTriple.Object.CopyNode(itemContent, false); + + itemContent.Assert(itemContentSubjectNode, itemContentPredicateNode, itemContentObjectNode); + + if (itemContentTriple.Object is IUriNode) + { + Utils.CopyCatalogContentGraph(itemContentTriple.Object, graph, itemContent); + } + } + + entries.Add(itemNode.Uri.AbsoluteUri, new CatalogItemSummary(type, commitId, timeStamp, count, itemContent)); + } + + return entries; + } + + private async Task SaveGraph(Uri resourceUri, IGraph graph, Uri typeUri, CancellationToken cancellationToken) + { + if (GraphPersistence != null) + { + await GraphPersistence.SaveGraph(resourceUri, graph, typeUri, cancellationToken); + } + else + { + await Storage.SaveAsync(resourceUri, CreateIndexContent(graph, typeUri), cancellationToken); + } + } + + private async Task LoadGraph(Uri resourceUri, CancellationToken cancellationToken) + { + if (GraphPersistence != null) + { + return await GraphPersistence.LoadGraph(resourceUri, cancellationToken); + } + else + { + return Utils.CreateGraph(resourceUri, await Storage.LoadStringAsync(resourceUri, cancellationToken)); + } + } + + protected Uri CreatePageUri(Uri baseAddress, string relativeAddress) + { + if (GraphPersistence != null) + { + return GraphPersistence.CreatePageUri(baseAddress, relativeAddress); + } + else + { + return new Uri(baseAddress, relativeAddress + ".json"); + } + } + + private void CheckScheme(Uri resourceUri, IGraph graph) + { + INode typePredicate = graph.CreateUriNode(Schema.Predicates.Type); + + Triple catalogRoot = graph.GetTriplesWithPredicateObject(typePredicate, graph.CreateUriNode(Schema.DataTypes.CatalogRoot)).FirstOrDefault(); + if (catalogRoot != null) + { + if (((UriNode)catalogRoot.Subject).Uri.Scheme != resourceUri.Scheme) + { + throw new ArgumentException("the resource scheme does not match the existing catalog"); + } + } + Triple catalogPage = graph.GetTriplesWithPredicateObject(typePredicate, graph.CreateUriNode(Schema.DataTypes.CatalogPage)).FirstOrDefault(); + if (catalogPage != null) + { + if (((UriNode)catalogPage.Subject).Uri.Scheme != resourceUri.Scheme) + { + throw new ArgumentException("the resource scheme does not match the existing catalog"); + } + } + } + } +} \ No newline at end of file diff --git a/src/Catalog/CloudBlobStorageExtensions.cs b/src/Catalog/CloudBlobStorageExtensions.cs new file mode 100644 index 000000000..f40a891d3 --- /dev/null +++ b/src/Catalog/CloudBlobStorageExtensions.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace NuGet.Services.Metadata.Catalog +{ + internal static class CloudBlobStorageExtensions + { + public static async Task> ListBlobsAsync( + this CloudBlobDirectory directory, CancellationToken cancellationToken) + { + var items = new List(); + BlobContinuationToken continuationToken = null; + do + { + var segment = await directory.ListBlobsSegmentedAsync( + useFlatBlobListing: true, + blobListingDetails: BlobListingDetails.None, + maxResults: null, + currentToken: continuationToken, + options: null, + operationContext: null, + cancellationToken: cancellationToken); + + continuationToken = segment.ContinuationToken; + items.AddRange(segment.Results); + } + while (continuationToken != null && !cancellationToken.IsCancellationRequested); + + return items; + } + } +} \ No newline at end of file diff --git a/src/Catalog/CollectorBase.cs b/src/Catalog/CollectorBase.cs new file mode 100644 index 000000000..952431a81 --- /dev/null +++ b/src/Catalog/CollectorBase.cs @@ -0,0 +1,79 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog +{ + public abstract class CollectorBase + { + protected readonly ITelemetryService _telemetryService; + private readonly Func _handlerFunc; + private readonly IHttpRetryStrategy _httpRetryStrategy; + private readonly TimeSpan? _httpClientTimeout; + + public CollectorBase( + Uri index, + ITelemetryService telemetryService, + Func handlerFunc = null, + TimeSpan? httpClientTimeout = null, + IHttpRetryStrategy httpRetryStrategy = null) + { + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _handlerFunc = handlerFunc; + _httpClientTimeout = httpClientTimeout; + _httpRetryStrategy = httpRetryStrategy; + Index = index ?? throw new ArgumentNullException(nameof(index)); + } + + public Uri Index { get; } + + public async Task RunAsync(CancellationToken cancellationToken) + { + return await RunAsync(MemoryCursor.CreateMin(), MemoryCursor.CreateMax(), cancellationToken); + } + + public async Task RunAsync(DateTime front, DateTime back, CancellationToken cancellationToken) + { + return await RunAsync(new MemoryCursor(front), new MemoryCursor(back), cancellationToken); + } + + public async Task RunAsync(ReadWriteCursor front, ReadCursor back, CancellationToken cancellationToken) + { + await Task.WhenAll(front.LoadAsync(cancellationToken), back.LoadAsync(cancellationToken)); + + Trace.TraceInformation("Run ( {0} , {1} )", front, back); + + bool result = false; + + HttpMessageHandler handler = null; + + if (_handlerFunc != null) + { + handler = _handlerFunc(); + } + + using (CollectorHttpClient client = new CollectorHttpClient(handler, _httpRetryStrategy)) + { + if (_httpClientTimeout.HasValue) + { + client.Timeout = _httpClientTimeout.Value; + } + + result = await FetchAsync(client, front, back, cancellationToken); + } + + return result; + } + + protected abstract Task FetchAsync( + CollectorHttpClient client, + ReadWriteCursor front, + ReadCursor back, + CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/Catalog/CollectorHttpClient.cs b/src/Catalog/CollectorHttpClient.cs new file mode 100644 index 000000000..6305a333d --- /dev/null +++ b/src/Catalog/CollectorHttpClient.cs @@ -0,0 +1,110 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using VDS.RDF; + +namespace NuGet.Services.Metadata.Catalog +{ + public class CollectorHttpClient : HttpClient + { + private int _requestCount; + private readonly IHttpRetryStrategy _retryStrategy; + + public CollectorHttpClient() + : this(new WebRequestHandler { AllowPipelining = true }) + { + } + + public CollectorHttpClient(HttpMessageHandler handler, IHttpRetryStrategy retryStrategy = null) + : base(handler ?? new WebRequestHandler { AllowPipelining = true }) + { + _requestCount = 0; + _retryStrategy = retryStrategy ?? new RetryWithExponentialBackoff(); + } + + public int RequestCount + { + get { return _requestCount; } + } + + protected void InReqCount() + { + Interlocked.Increment(ref _requestCount); + } + + public virtual Task GetJObjectAsync(Uri address) + { + return GetJObjectAsync(address, CancellationToken.None); + } + + public virtual async Task GetJObjectAsync(Uri address, CancellationToken token) + { + InReqCount(); + + var json = await GetStringAsync(address, token); + + try + { + return ParseJObject(json); + } + catch (Exception e) + { + throw new Exception($"{nameof(GetJObjectAsync)}({address})", e); + } + } + + private static JObject ParseJObject(string json) + { + using (var reader = new JsonTextReader(new StringReader(json))) + { + reader.DateParseHandling = DateParseHandling.DateTimeOffset; // make sure we always preserve timezone info + + return JObject.Load(reader); + } + } + + public virtual Task GetGraphAsync(Uri address) + { + return GetGraphAsync(address, readOnly: false, token: CancellationToken.None); + } + + public virtual Task GetGraphAsync(Uri address, bool readOnly, CancellationToken token) + { + var task = GetJObjectAsync(address, token); + + return task.ContinueWith((t) => + { + try + { + return Utils.CreateGraph(t.Result, readOnly); + } + catch (Exception e) + { + throw new Exception($"{nameof(GetGraphAsync)}({address})", e); + } + }, token); + } + + public virtual async Task GetStringAsync(Uri address, CancellationToken token) + { + try + { + using (var httpResponse = await _retryStrategy.SendAsync(this, address, token)) + { + return await httpResponse.Content.ReadAsStringAsync(); + } + } + catch (Exception e) + { + throw new Exception($"{nameof(GetStringAsync)}({address})", e); + } + } + } +} \ No newline at end of file diff --git a/src/Catalog/CommitCollector.cs b/src/Catalog/CommitCollector.cs new file mode 100644 index 000000000..a120bbe45 --- /dev/null +++ b/src/Catalog/CommitCollector.cs @@ -0,0 +1,199 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; + +namespace NuGet.Services.Metadata.Catalog +{ + public abstract class CommitCollector : CollectorBase + { + public CommitCollector( + Uri index, + ITelemetryService telemetryService, + Func handlerFunc = null, + TimeSpan? httpClientTimeout = null, + IHttpRetryStrategy httpRetryStrategy = null) + : base(index, telemetryService, handlerFunc, httpClientTimeout, httpRetryStrategy) + { + } + + protected override async Task FetchAsync( + CollectorHttpClient client, + ReadWriteCursor front, + ReadCursor back, + CancellationToken cancellationToken) + { + var commits = await FetchCatalogCommitsAsync(client, front, back, cancellationToken); + + bool acceptNextBatch = false; + + foreach (CatalogCommit commit in commits) + { + JObject page = await client.GetJObjectAsync(commit.Uri, cancellationToken); + + JToken context = null; + page.TryGetValue("@context", out context); + + var batches = await CreateBatchesAsync(page["items"] + .Select(item => CatalogCommitItem.Create((JObject)context, (JObject)item)) + .Where(item => item.CommitTimeStamp > front.Value && item.CommitTimeStamp <= back.Value)); + + var orderedBatches = batches + .OrderBy(batch => batch.CommitTimeStamp) + .ToList(); + + var lastBatch = orderedBatches.LastOrDefault(); + DateTime? previousCommitTimeStamp = null; + + foreach (var batch in orderedBatches) + { + // If the commit timestamp has changed from the previous batch, commit. This is important because if + // two batches have the same commit timestamp but processing the second fails, we should not + // progress the cursor forward. + if (previousCommitTimeStamp.HasValue && previousCommitTimeStamp != batch.CommitTimeStamp) + { + front.Value = previousCommitTimeStamp.Value; + await front.SaveAsync(cancellationToken); + Trace.TraceInformation("CommitCatalog.Fetch front.Value saved since timestamp changed from previous: {0}", front); + } + + using (_telemetryService.TrackDuration(TelemetryConstants.ProcessBatchSeconds, new Dictionary() + { + { TelemetryConstants.BatchItemCount, batch.Items.Count.ToString() } + })) + { + acceptNextBatch = await OnProcessBatchAsync( + client, + batch.Items, + context, + batch.CommitTimeStamp, + batch.CommitTimeStamp == lastBatch.CommitTimeStamp, + cancellationToken); + } + + // If this is the last batch, commit the cursor. + if (ReferenceEquals(batch, lastBatch)) + { + front.Value = batch.CommitTimeStamp; + await front.SaveAsync(cancellationToken); + Trace.TraceInformation("CommitCatalog.Fetch front.Value saved due to last batch: {0}", front); + } + + previousCommitTimeStamp = batch.CommitTimeStamp; + + Trace.TraceInformation("CommitCatalog.Fetch front.Value is: {0}", front); + + if (!acceptNextBatch) + { + break; + } + } + + if (!acceptNextBatch) + { + break; + } + } + + return acceptNextBatch; + } + + protected async Task> FetchCatalogCommitsAsync( + CollectorHttpClient client, + ReadCursor front, + ReadCursor back, + CancellationToken cancellationToken) + { + JObject root; + + using (_telemetryService.TrackDuration( + TelemetryConstants.CatalogIndexReadDurationSeconds, + new Dictionary() { { TelemetryConstants.Uri, Index.AbsoluteUri } })) + { + root = await client.GetJObjectAsync(Index, cancellationToken); + } + + var commits = root["items"].Select(item => CatalogCommit.Create((JObject)item)); + return GetCommitsInRange(commits, front.Value, back.Value); + } + + + public static IEnumerable GetCommitsInRange( + IEnumerable commits, + DateTimeOffset minCommitTimestamp, + DateTimeOffset maxCommitTimestamp) + { + // Only consider pages that have a (latest) commit timestamp greater than the minimum bound. If a page has + // a commit timestamp greater than the minimum bound, then there is at least one item with a commit + // timestamp greater than the minimum bound. Sort the pages by commit timestamp so that they are + // in chronological order. + var upperRange = commits + .Where(x => x.CommitTimeStamp > minCommitTimestamp) + .OrderBy(x => x.CommitTimeStamp); + + // Take pages from the sorted list until the (latest) commit timestamp goes past the maximum commit + // timestamp. This essentially LINQ's TakeWhile plus one more element. Because the maximum bound is + // inclusive, we need to yield any page that has a (latest) commit timestamp that is less than or + // equal to the maximum bound. + // + // Consider the following pages (bounded by square brackets, labeled P-0 ... P-N) containing commits + // (C-0 ... C-N). The front cursor (exclusive minimum bound) is marked by the letter "F" and the back + // cursor (inclusive upper bound) is marked by the letter "B". + // + // ---- P-0 ---- ---- P-1 ---- ---- P-2 ---- ----- P-3 ----- + // [ C-0, C-1, C-2 ] [ C-3, C-4, C-5 ] [ C-6, C-7, C-8 ] [ C-9, C-10, C-11 ] + // | | | | | | + // Scenario #1: F | | | B | + // | | | | + // P-0, P-1, and P-2 should be downloaded and C-2 to C-7 should be processed. Note that P-3 should not + // even be considered because P-2 is the first page with a maximum commit timestamp greater than "B". + // | | | | + // Scenario #2: | F | B + // | | + // P-1 and P-2 should be downloaded and C-4 to C-8 should be processed. The concept of a timestamp-based + // cursor requires that commit timestamps strictly increase. Additionally, our catalog implementation + // never allows a commit to be split across multiple pages. In other words, if C-8 is at the end of P-2 + // the consumer of the catalog can assume that P-3 only has commits later than C-8 and therefore P-3 need + // not be considered. | + // | | + // Scenario #3: F B + // + // P-1 and P-2 should be downloaded and C-3 to C-6 should be processed. Because "F" (an exclusive bound) + // is pointing to the latest commit timestamp in P-0, that page can be completely ignored. + foreach (var page in upperRange) + { + yield return page; + + if (page.CommitTimeStamp >= maxCommitTimestamp) + { + break; + } + } + } + + protected virtual Task> CreateBatchesAsync(IEnumerable catalogItems) + { + var batches = catalogItems + .GroupBy(item => item.CommitTimeStamp) + .OrderBy(group => group.Key) + .Select(group => new CatalogCommitItemBatch(group)); + + return Task.FromResult(batches); + } + + protected abstract Task OnProcessBatchAsync( + CollectorHttpClient client, + IEnumerable items, + JToken context, + DateTime commitTimeStamp, + bool isLastBatch, + CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/Catalog/CommitMetadata.cs b/src/Catalog/CommitMetadata.cs new file mode 100644 index 000000000..687827dfe --- /dev/null +++ b/src/Catalog/CommitMetadata.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog +{ + public class CommitMetadata + { + public CommitMetadata() + { + } + + public CommitMetadata(DateTime? lastCreated, DateTime? lastEdited, DateTime? lastDeleted) + { + LastCreated = lastCreated; + LastEdited = lastEdited; + LastDeleted = lastDeleted; + } + + public DateTime? LastCreated { get; set; } + public DateTime? LastEdited { get; set; } + public DateTime? LastDeleted { get; set; } + } +} \ No newline at end of file diff --git a/src/Catalog/Constants.cs b/src/Catalog/Constants.cs new file mode 100644 index 000000000..72e42dace --- /dev/null +++ b/src/Catalog/Constants.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog +{ + public static class Constants + { + public static readonly DateTime DateTimeMinValueUtc = DateTimeOffset.MinValue.UtcDateTime; + public const int MaxPageSize = 550; + public const string Sha512 = "SHA512"; + public static readonly DateTime UnpublishedDate = new DateTime(1900, 1, 1, 0, 0, 0, DateTimeKind.Utc); + } +} \ No newline at end of file diff --git a/src/Catalog/CreateCommitItemBatchesAsync.cs b/src/Catalog/CreateCommitItemBatchesAsync.cs new file mode 100644 index 000000000..d95efee77 --- /dev/null +++ b/src/Catalog/CreateCommitItemBatchesAsync.cs @@ -0,0 +1,11 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog +{ + internal delegate Task> CreateCommitItemBatchesAsync( + IEnumerable catalogItems); +} \ No newline at end of file diff --git a/src/Catalog/DeleteCatalogItem.cs b/src/Catalog/DeleteCatalogItem.cs new file mode 100644 index 000000000..ca7c2cc7c --- /dev/null +++ b/src/Catalog/DeleteCatalogItem.cs @@ -0,0 +1,118 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using Newtonsoft.Json.Linq; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Versioning; +using VDS.RDF; + +namespace NuGet.Services.Metadata.Catalog +{ + public class DeleteCatalogItem : AppendOnlyCatalogItem + { + private string _id; + private string _version; + private readonly DateTime _published; + + public DeleteCatalogItem(string id, string version, DateTime published) + { + _id = id; + _version = TryNormalize(version); + _published = published; + } + + private string TryNormalize(string version) + { + SemanticVersion semVer; + if (SemanticVersion.TryParse(version, out semVer)) + { + return semVer.ToNormalizedString(); + } + return version; + } + + public override Uri GetItemType() + { + return Schema.DataTypes.PackageDelete; + } + + protected override string GetItemIdentity() + { + return (_id + "." + _version).ToLowerInvariant(); + } + + public override StorageContent CreateContent(CatalogContext context) + { + using (IGraph graph = new Graph()) + { + INode entry = graph.CreateUriNode(GetItemAddress()); + + // catalog infrastructure fields + graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.Type), graph.CreateUriNode(GetItemType())); + graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.Type), graph.CreateUriNode(Schema.DataTypes.Permalink)); + graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.CatalogTimeStamp), graph.CreateLiteralNode(TimeStamp.ToString("O"), Schema.DataTypes.DateTime)); + graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.CatalogCommitId), graph.CreateLiteralNode(CommitId.ToString())); + + graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.Published), graph.CreateLiteralNode(_published.ToString("O"), Schema.DataTypes.DateTime)); + + graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.Id), graph.CreateLiteralNode(_id)); + graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.OriginalId), graph.CreateLiteralNode(_id)); + graph.Assert(entry, graph.CreateUriNode(Schema.Predicates.Version), graph.CreateLiteralNode(_version)); + + SetIdVersionFromGraph(graph); + + // create JSON content + JObject frame = context.GetJsonLdContext("context.Catalog.json", GetItemType()); + StorageContent content = new StringStorageContent(Utils.CreateArrangedJson(graph, frame), "application/json", "no-store"); + + return content; + } + } + + public override IGraph CreatePageContent(CatalogContext context) + { + var resourceUri = new Uri(GetBaseAddress() + GetRelativeAddress()); + + var graph = new Graph(); + + var subject = graph.CreateUriNode(resourceUri); + + var idPredicate = graph.CreateUriNode(Schema.Predicates.Id); + var versionPredicate = graph.CreateUriNode(Schema.Predicates.Version); + + if (_id != null) + { + graph.Assert(subject, idPredicate, graph.CreateLiteralNode(_id)); + } + + if (_version != null) + { + graph.Assert(subject, versionPredicate, graph.CreateLiteralNode(_version)); + } + + return graph; + } + + private void SetIdVersionFromGraph(IGraph graph) + { + var resource = graph.GetTriplesWithPredicateObject( + graph.CreateUriNode(Schema.Predicates.Type), graph.CreateUriNode(GetItemType())).First(); + + var id = graph.GetTriplesWithSubjectPredicate( + resource.Subject, graph.CreateUriNode(Schema.Predicates.Id)).FirstOrDefault(); + if (id != null) + { + _id = ((ILiteralNode)id.Object).Value; + } + + var version = graph.GetTriplesWithSubjectPredicate( + resource.Subject, graph.CreateUriNode(Schema.Predicates.Version)).FirstOrDefault(); + if (version != null) + { + _version = ((ILiteralNode)version.Object).Value; + } + } + } +} \ No newline at end of file diff --git a/src/Catalog/Dnx/DnxCatalogCollector.cs b/src/Catalog/Dnx/DnxCatalogCollector.cs new file mode 100644 index 000000000..2f06bf2e1 --- /dev/null +++ b/src/Catalog/Dnx/DnxCatalogCollector.cs @@ -0,0 +1,564 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json.Linq; +using NuGet.Protocol.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Versioning; + +namespace NuGet.Services.Metadata.Catalog.Dnx +{ + public class DnxCatalogCollector : CommitCollector + { + private readonly StorageFactory _storageFactory; + private readonly IAzureStorage _sourceStorage; + private readonly DnxMaker _dnxMaker; + private readonly Func _catalogClientFactory; + private readonly ILogger _logger; + private readonly int _maxConcurrentBatches; + private readonly int _maxConcurrentCommitItemsWithinBatch; + private readonly Uri _contentBaseAddress; + + public DnxCatalogCollector( + Uri index, + StorageFactory storageFactory, + IAzureStorage preferredPackageSourceStorage, + Uri contentBaseAddress, + ITelemetryService telemetryService, + ILogger logger, + int maxDegreeOfParallelism, + Func catalogClientFactory, + Func handlerFunc = null, + TimeSpan? httpClientTimeout = null) + : base(index, telemetryService, handlerFunc, httpClientTimeout) + { + _storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory)); + _sourceStorage = preferredPackageSourceStorage; + _contentBaseAddress = contentBaseAddress; + _dnxMaker = new DnxMaker(storageFactory, telemetryService, logger); + _catalogClientFactory = catalogClientFactory ?? throw new ArgumentNullException(nameof(catalogClientFactory)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (maxDegreeOfParallelism < 1) + { + throw new ArgumentOutOfRangeException( + nameof(maxDegreeOfParallelism), + string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue)); + } + + // Find two factors which are close or equal to each other. + var squareRoot = Math.Sqrt(maxDegreeOfParallelism); + + // If the max degree of parallelism is a perfect square, great. + // Otherwise, prefer a greater degree of parallelism in batches than commit items within a batch. + _maxConcurrentBatches = Convert.ToInt32(Math.Ceiling(squareRoot)); + _maxConcurrentCommitItemsWithinBatch = Convert.ToInt32(maxDegreeOfParallelism / _maxConcurrentBatches); + + ServicePointManager.DefaultConnectionLimit = _maxConcurrentBatches * _maxConcurrentCommitItemsWithinBatch; + } + + protected override Task> CreateBatchesAsync( + IEnumerable catalogItems) + { + var batches = CatalogCommitUtilities.CreateCommitItemBatches( + catalogItems, + CatalogCommitUtilities.GetPackageIdKey); + + return Task.FromResult(batches); + } + + protected override Task FetchAsync( + CollectorHttpClient client, + ReadWriteCursor front, + ReadCursor back, + CancellationToken cancellationToken) + { + return CatalogCommitUtilities.ProcessCatalogCommitsAsync( + client, + front, + back, + FetchCatalogCommitsAsync, + CreateBatchesAsync, + ProcessBatchAsync, + _maxConcurrentBatches, + _logger, + cancellationToken); + } + + protected override async Task OnProcessBatchAsync( + CollectorHttpClient client, + IEnumerable items, + JToken context, + DateTime commitTimeStamp, + bool isLastBatch, + CancellationToken cancellationToken) + { + var catalogEntries = items.Select(item => CatalogEntry.Create(item)) + .ToList(); + + // Sanity check: a single catalog batch should not contain multiple entries for the same package identity. + AssertNoMultipleEntriesForSamePackageIdentity(commitTimeStamp, catalogEntries); + + // Process .nupkg/.nuspec adds and deletes. + var processedCatalogEntries = await ProcessCatalogEntriesAsync(client, catalogEntries, cancellationToken); + + // Update the package version index with adds and deletes. + await UpdatePackageVersionIndexAsync(processedCatalogEntries, cancellationToken); + + return true; + } + + private async Task> ProcessCatalogEntriesAsync( + CollectorHttpClient client, + IEnumerable catalogEntries, + CancellationToken cancellationToken) + { + var processedCatalogEntries = new ConcurrentBag(); + + await catalogEntries.ForEachAsync(_maxConcurrentCommitItemsWithinBatch, async catalogEntry => + { + var packageId = catalogEntry.PackageId; + var normalizedPackageVersion = catalogEntry.NormalizedPackageVersion; + + if (catalogEntry.Type.AbsoluteUri == Schema.DataTypes.PackageDetails.AbsoluteUri) + { + var telemetryProperties = GetTelemetryProperties(catalogEntry); + + using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageDetailsSeconds, telemetryProperties)) + { + var packageFileName = PackageUtility.GetPackageFileName( + packageId, + normalizedPackageVersion); + var sourceUri = new Uri(_contentBaseAddress, packageFileName); + var destinationStorage = _storageFactory.Create(packageId); + var destinationRelativeUri = DnxMaker.GetRelativeAddressNupkg( + packageId, + normalizedPackageVersion); + var destinationUri = destinationStorage.GetUri(destinationRelativeUri); + + var isNupkgSynchronized = await destinationStorage.AreSynchronized(sourceUri, destinationUri); + var isPackageInIndex = await _dnxMaker.HasPackageInIndexAsync( + destinationStorage, + packageId, + normalizedPackageVersion, + cancellationToken); + var areRequiredPropertiesPresent = await AreRequiredPropertiesPresentAsync(destinationStorage, destinationUri); + + if (isNupkgSynchronized && isPackageInIndex && areRequiredPropertiesPresent) + { + _logger.LogInformation("No changes detected: {Id}/{Version}", packageId, normalizedPackageVersion); + + return; + } + + if ((isNupkgSynchronized && areRequiredPropertiesPresent) + || await ProcessPackageDetailsAsync( + client, + packageId, + normalizedPackageVersion, + sourceUri, + catalogEntry.Uri, + telemetryProperties, + cancellationToken)) + { + processedCatalogEntries.Add(catalogEntry); + } + } + } + else if (catalogEntry.Type.AbsoluteUri == Schema.DataTypes.PackageDelete.AbsoluteUri) + { + var properties = GetTelemetryProperties(catalogEntry); + + using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageDeleteSeconds, properties)) + { + await ProcessPackageDeleteAsync(packageId, normalizedPackageVersion, cancellationToken); + + processedCatalogEntries.Add(catalogEntry); + } + } + }); + + return processedCatalogEntries; + } + + private async Task AreRequiredPropertiesPresentAsync(Storage destinationStorage, Uri destinationUri) + { + var azureStorage = destinationStorage as IAzureStorage; + + if (azureStorage == null) + { + return true; + } + + return await azureStorage.HasPropertiesAsync( + destinationUri, + DnxConstants.ApplicationOctetStreamContentType, + DnxConstants.DefaultCacheControl); + } + + private async Task UpdatePackageVersionIndexAsync( + IEnumerable catalogEntries, + CancellationToken cancellationToken) + { + var catalogEntryGroups = catalogEntries.GroupBy(catalogEntry => catalogEntry.PackageId); + + await catalogEntryGroups.ForEachAsync(_maxConcurrentCommitItemsWithinBatch, async catalogEntryGroup => + { + var packageId = catalogEntryGroup.Key; + var properties = new Dictionary() + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.BatchItemCount, catalogEntryGroup.Count().ToString() } + }; + + using (_telemetryService.TrackDuration(TelemetryConstants.ProcessPackageVersionIndexSeconds, properties)) + { + await _dnxMaker.UpdatePackageVersionIndexAsync(packageId, versions => + { + foreach (var catalogEntry in catalogEntryGroup) + { + if (catalogEntry.Type.AbsoluteUri == Schema.DataTypes.PackageDetails.AbsoluteUri) + { + versions.Add(NuGetVersion.Parse(catalogEntry.NormalizedPackageVersion)); + } + else if (catalogEntry.Type.AbsoluteUri == Schema.DataTypes.PackageDelete.AbsoluteUri) + { + versions.Remove(NuGetVersion.Parse(catalogEntry.NormalizedPackageVersion)); + } + } + }, cancellationToken); + } + + foreach (var catalogEntry in catalogEntryGroup) + { + _logger.LogInformation("Commit: {Id}/{Version}", packageId, catalogEntry.NormalizedPackageVersion); + } + }); + } + + private async Task ProcessPackageDetailsAsync( + HttpClient client, + string packageId, + string normalizedPackageVersion, + Uri sourceUri, + Uri catalogLeafUri, + Dictionary telemetryProperties, + CancellationToken cancellationToken) + { + var catalogClient = _catalogClientFactory(client); + var catalogLeaf = await catalogClient.GetPackageDetailsLeafAsync(catalogLeafUri.AbsoluteUri); + + if (await ProcessPackageDetailsViaStorageAsync( + packageId, + normalizedPackageVersion, + catalogLeaf, + telemetryProperties, + cancellationToken)) + { + return true; + } + + _telemetryService.TrackMetric( + TelemetryConstants.UsePackageSourceFallback, + metric: 1, + properties: GetTelemetryProperties(packageId, normalizedPackageVersion)); + + return await ProcessPackageDetailsViaHttpAsync( + client, + packageId, + normalizedPackageVersion, + sourceUri, + catalogLeaf, + telemetryProperties, + cancellationToken); + } + + private async Task ProcessPackageDetailsViaStorageAsync( + string packageId, + string normalizedPackageVersion, + PackageDetailsCatalogLeaf catalogLeaf, + Dictionary telemetryProperties, + CancellationToken cancellationToken) + { + if (_sourceStorage == null) + { + return false; + } + + var packageFileName = PackageUtility.GetPackageFileName(packageId, normalizedPackageVersion); + var sourceUri = _sourceStorage.ResolveUri(packageFileName); + + var sourceBlob = await _sourceStorage.GetCloudBlockBlobReferenceAsync(sourceUri); + + if (await sourceBlob.ExistsAsync(cancellationToken)) + { + // It's possible (though unlikely) that the blob may change between reads. Reading a blob with a + // single GET request returns the whole blob in a consistent state, but we're reading the blob many + // different times. To detect the blob changing between reads, we check the ETag again later. + // If the ETag's differ, we'll fall back to using a single HTTP GET request. + var token1 = await _sourceStorage.GetOptimisticConcurrencyControlTokenAsync(sourceUri, cancellationToken); + + telemetryProperties[TelemetryConstants.SizeInBytes] = sourceBlob.Length.ToString(); + + var nuspec = await GetNuspecAsync(sourceBlob, packageId, cancellationToken); + + if (string.IsNullOrEmpty(nuspec)) + { + _logger.LogWarning( + "No .nuspec available for {Id}/{Version}. Falling back to HTTP processing.", + packageId, + normalizedPackageVersion); + } + else + { + await _dnxMaker.AddPackageAsync( + _sourceStorage, + nuspec, + packageId, + normalizedPackageVersion, + catalogLeaf.IconFile, + cancellationToken); + + var token2 = await _sourceStorage.GetOptimisticConcurrencyControlTokenAsync(sourceUri, cancellationToken); + + if (token1 == token2) + { + _logger.LogInformation("Added .nupkg and .nuspec for package {Id}/{Version}", packageId, normalizedPackageVersion); + + return true; + } + else + { + _telemetryService.TrackMetric( + TelemetryConstants.BlobModified, + metric: 1, + properties: GetTelemetryProperties(packageId, normalizedPackageVersion)); + } + } + } + else + { + _telemetryService.TrackMetric( + TelemetryConstants.NonExistentBlob, + metric: 1, + properties: GetTelemetryProperties(packageId, normalizedPackageVersion)); + } + + return false; + } + + private async Task ProcessPackageDetailsViaHttpAsync( + HttpClient client, + string id, + string version, + Uri sourceUri, + PackageDetailsCatalogLeaf catalogLeaf, + Dictionary telemetryProperties, + CancellationToken cancellationToken) + { + var packageDownloader = new PackageDownloader(client, _logger); + var requestUri = Utilities.GetNugetCacheBustingUri(sourceUri); + + using (var stream = await packageDownloader.DownloadAsync(requestUri, cancellationToken)) + { + if (stream == null) + { + _logger.LogWarning("Package {Id}/{Version} not found.", id, version); + + return false; + } + + telemetryProperties[TelemetryConstants.SizeInBytes] = stream.Length.ToString(); + + var nuspec = GetNuspec(stream, id); + + if (nuspec == null) + { + _logger.LogWarning("No .nuspec available for {Id}/{Version}. Skipping.", id, version); + + return false; + } + + stream.Position = 0; + + await _dnxMaker.AddPackageAsync( + stream, + nuspec, + id, + version, + catalogLeaf.IconFile, + cancellationToken); + } + + _logger.LogInformation("Added .nupkg and .nuspec for package {Id}/{Version}", id, version); + + return true; + } + + private async Task ProcessPackageDeleteAsync( + string packageId, + string normalizedPackageVersion, + CancellationToken cancellationToken) + { + await _dnxMaker.UpdatePackageVersionIndexAsync( + packageId, + versions => versions.Remove(NuGetVersion.Parse(normalizedPackageVersion)), + cancellationToken); + + await _dnxMaker.DeletePackageAsync(packageId, normalizedPackageVersion, cancellationToken); + + _logger.LogInformation("Commit delete: {Id}/{Version}", packageId, normalizedPackageVersion); + } + + private static void AssertNoMultipleEntriesForSamePackageIdentity( + DateTime commitTimeStamp, + IEnumerable catalogEntries) + { + var catalogEntriesForSamePackageIdentity = catalogEntries.GroupBy( + catalogEntry => new + { + catalogEntry.PackageId, + catalogEntry.NormalizedPackageVersion + }) + .Where(group => group.Count() > 1) + .Select(group => $"{group.Key.PackageId} {group.Key.NormalizedPackageVersion}"); + + if (catalogEntriesForSamePackageIdentity.Any()) + { + var packageIdentities = string.Join(", ", catalogEntriesForSamePackageIdentity); + + throw new InvalidOperationException($"The catalog batch {commitTimeStamp} contains multiple entries for the same package identity. Package(s): {packageIdentities}"); + } + } + + private async Task GetNuspecAsync( + ICloudBlockBlob sourceBlob, + string packageId, + CancellationToken cancellationToken) + { + using (var stream = await sourceBlob.GetStreamAsync(cancellationToken)) + { + return GetNuspec(stream, packageId); + } + } + + private static string GetNuspec(Stream stream, string id) + { + string name = $"{id}.nuspec"; + + using (var archive = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true)) + { + // first look for a nuspec file named as the package id + foreach (ZipArchiveEntry entry in archive.Entries) + { + if (entry.FullName.Equals(name, StringComparison.InvariantCultureIgnoreCase)) + { + using (TextReader reader = new StreamReader(entry.Open())) + { + return reader.ReadToEnd(); + } + } + } + // failing that, just return the first file that appears to be a nuspec + foreach (ZipArchiveEntry entry in archive.Entries) + { + if (entry.FullName.EndsWith(".nuspec", StringComparison.InvariantCultureIgnoreCase)) + { + using (TextReader reader = new StreamReader(entry.Open())) + { + return reader.ReadToEnd(); + } + } + } + } + + return null; + } + + private static Dictionary GetTelemetryProperties(CatalogEntry catalogEntry) + { + return GetTelemetryProperties(catalogEntry.PackageId, catalogEntry.NormalizedPackageVersion); + } + + private static Dictionary GetTelemetryProperties(string packageId, string normalizedPackageVersion) + { + return new Dictionary() + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, normalizedPackageVersion } + }; + } + + private async Task ProcessBatchAsync( + CollectorHttpClient client, + JToken context, + string packageId, + CatalogCommitItemBatch batch, + CatalogCommitItemBatch lastBatch, + CancellationToken cancellationToken) + { + await Task.Yield(); + + using (_telemetryService.TrackDuration( + TelemetryConstants.ProcessBatchSeconds, + new Dictionary() + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.BatchItemCount, batch.Items.Count.ToString() } + })) + { + await OnProcessBatchAsync( + client, + batch.Items, + context, + batch.CommitTimeStamp, + isLastBatch: false, + cancellationToken: cancellationToken); + } + } + + private sealed class CatalogEntry + { + internal DateTime CommitTimeStamp { get; } + internal string PackageId { get; } + internal string NormalizedPackageVersion { get; } + internal Uri Type { get; } + internal Uri Uri { get; } + + private CatalogEntry(DateTime commitTimeStamp, string packageId, string normalizedPackageVersion, Uri type, Uri uri) + { + CommitTimeStamp = commitTimeStamp; + PackageId = packageId; + NormalizedPackageVersion = normalizedPackageVersion; + Type = type; + Uri = uri; + } + + internal static CatalogEntry Create(CatalogCommitItem item) + { + var typeUri = item.TypeUris.Single(uri => + uri.AbsoluteUri == Schema.DataTypes.PackageDetails.AbsoluteUri || + uri.AbsoluteUri == Schema.DataTypes.PackageDelete.AbsoluteUri); + + return new CatalogEntry( + item.CommitTimeStamp, + item.PackageIdentity.Id.ToLowerInvariant(), + item.PackageIdentity.Version.ToNormalizedString().ToLowerInvariant(), + typeUri, + item.Uri); + } + } + } +} \ No newline at end of file diff --git a/src/Catalog/Dnx/DnxConstants.cs b/src/Catalog/Dnx/DnxConstants.cs new file mode 100644 index 000000000..385f520a7 --- /dev/null +++ b/src/Catalog/Dnx/DnxConstants.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog.Dnx +{ + internal static class DnxConstants + { + internal const string ApplicationOctetStreamContentType = "application/octet-stream"; + internal const string DefaultCacheControl = "max-age=120"; + + internal static readonly IReadOnlyDictionary RequiredBlobProperties = new Dictionary() + { + { StorageConstants.CacheControl, DefaultCacheControl }, + { StorageConstants.ContentType, ApplicationOctetStreamContentType } + }; + } +} \ No newline at end of file diff --git a/src/Catalog/Dnx/DnxEntry.cs b/src/Catalog/Dnx/DnxEntry.cs new file mode 100644 index 000000000..9c8eac3a4 --- /dev/null +++ b/src/Catalog/Dnx/DnxEntry.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Dnx +{ + public sealed class DnxEntry + { + public Uri Nupkg { get; } + public Uri Nuspec { get; } + + internal DnxEntry(Uri nupkg, Uri nuspec) + { + Nupkg = nupkg; + Nuspec = nuspec; + } + } +} \ No newline at end of file diff --git a/src/Catalog/Dnx/DnxMaker.cs b/src/Catalog/Dnx/DnxMaker.cs new file mode 100644 index 000000000..d4ddaabb7 --- /dev/null +++ b/src/Catalog/Dnx/DnxMaker.cs @@ -0,0 +1,482 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NuGet.Common; +using Microsoft.Extensions.Logging; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Versioning; + +using ILogger = Microsoft.Extensions.Logging.ILogger; + +namespace NuGet.Services.Metadata.Catalog.Dnx +{ + public class DnxMaker + { + private readonly StorageFactory _storageFactory; + private readonly ITelemetryService _telemetryService; + private readonly ILogger _logger; + + public DnxMaker(StorageFactory storageFactory, ITelemetryService telemetryService, ILogger logger) + { + _storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task AddPackageAsync( + Stream nupkgStream, + string nuspec, + string packageId, + string normalizedPackageVersion, + string iconFilename, + CancellationToken cancellationToken) + { + if (nupkgStream == null) + { + throw new ArgumentNullException(nameof(nupkgStream)); + } + + if (!nupkgStream.CanSeek) + { + throw new ArgumentException($"{nameof(nupkgStream)} must be seekable stream", nameof(nupkgStream)); + } + + if (string.IsNullOrEmpty(nuspec)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(nuspec)); + } + + if (string.IsNullOrEmpty(packageId)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageId)); + } + + if (string.IsNullOrEmpty(normalizedPackageVersion)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(normalizedPackageVersion)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var storage = _storageFactory.Create(packageId); + var nuspecUri = await SaveNuspecAsync(storage, packageId, normalizedPackageVersion, nuspec, cancellationToken); + nupkgStream.Seek(0, SeekOrigin.Begin); + if (!string.IsNullOrWhiteSpace(iconFilename)) + { + await CopyIconFromNupkgStreamAsync(nupkgStream, iconFilename, storage, packageId, normalizedPackageVersion, cancellationToken); + } + else + { + _logger.LogInformation("Package {PackageId} {PackageVersion} doesn't have an icon file specified in fallback to package stream case.", + packageId, + normalizedPackageVersion); + } + var nupkgUri = await SaveNupkgAsync(nupkgStream, storage, packageId, normalizedPackageVersion, cancellationToken); + + return new DnxEntry(nupkgUri, nuspecUri); + } + + public async Task AddPackageAsync( + IAzureStorage sourceStorage, + string nuspec, + string packageId, + string normalizedPackageVersion, + string iconFilename, + CancellationToken cancellationToken) + { + if (sourceStorage == null) + { + throw new ArgumentNullException(nameof(sourceStorage)); + } + + if (string.IsNullOrEmpty(nuspec)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(nuspec)); + } + + if (string.IsNullOrEmpty(packageId)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageId)); + } + + if (string.IsNullOrEmpty(normalizedPackageVersion)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(normalizedPackageVersion)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var destinationStorage = _storageFactory.Create(packageId); + var nuspecUri = await SaveNuspecAsync(destinationStorage, packageId, normalizedPackageVersion, nuspec, cancellationToken); + if (!string.IsNullOrWhiteSpace(iconFilename)) + { + await CopyIconFromAzureStorageIfExistAsync(sourceStorage, destinationStorage, packageId, normalizedPackageVersion, iconFilename, cancellationToken); + } + else + { + _logger.LogInformation("Package {PackageId} {PackageVersion} doesn't have icon file specified in Azure Storage stream case", + packageId, + normalizedPackageVersion); + } + var nupkgUri = await CopyNupkgAsync(sourceStorage, destinationStorage, packageId, normalizedPackageVersion, cancellationToken); + + return new DnxEntry(nupkgUri, nuspecUri); + } + + public async Task DeletePackageAsync(string id, string version, CancellationToken cancellationToken) + { + if (string.IsNullOrEmpty(id)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(id)); + } + + if (string.IsNullOrEmpty(version)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(version)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var storage = _storageFactory.Create(id); + var normalizedVersion = NuGetVersionUtility.NormalizeVersion(version); + + await DeleteNuspecAsync(storage, id, normalizedVersion, cancellationToken); + await DeleteIconAsync(storage, id, normalizedVersion, cancellationToken); + await DeleteNupkgAsync(storage, id, normalizedVersion, cancellationToken); + } + + public async Task HasPackageInIndexAsync(Storage storage, string id, string version, CancellationToken cancellationToken) + { + if (storage == null) + { + throw new ArgumentNullException(nameof(storage)); + } + + if (string.IsNullOrEmpty(id)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(id)); + } + + if (string.IsNullOrEmpty(version)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(version)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var versionsContext = await GetVersionsAsync(storage, cancellationToken); + var parsedVersion = NuGetVersion.Parse(version); + + return versionsContext.Versions.Contains(parsedVersion); + } + + private async Task SaveNuspecAsync(Storage storage, string id, string version, string nuspec, CancellationToken cancellationToken) + { + var relativeAddress = GetRelativeAddressNuspec(id, version); + var nuspecUri = new Uri(storage.BaseAddress, relativeAddress); + var content = new StringStorageContent(nuspec, "text/xml", DnxConstants.DefaultCacheControl); + + await storage.SaveAsync(nuspecUri, content, cancellationToken); + + return nuspecUri; + } + + public async Task UpdatePackageVersionIndexAsync(string id, Action> updateAction, CancellationToken cancellationToken) + { + if (string.IsNullOrEmpty(id)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(id)); + } + + if (updateAction == null) + { + throw new ArgumentNullException(nameof(updateAction)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var storage = _storageFactory.Create(id); + var versionsContext = await GetVersionsAsync(storage, cancellationToken); + var relativeAddress = versionsContext.RelativeAddress; + var resourceUri = versionsContext.ResourceUri; + var versions = versionsContext.Versions; + + updateAction(versions); + var result = new List(versions); + + if (result.Any()) + { + // Store versions (sorted) + result.Sort(); + + await storage.SaveAsync(resourceUri, CreateContent(result.Select(version => version.ToNormalizedString())), cancellationToken); + } + else + { + // Remove versions file if no versions are present + if (storage.Exists(relativeAddress)) + { + await storage.DeleteAsync(resourceUri, cancellationToken); + } + } + } + + private async Task GetVersionsAsync(Storage storage, CancellationToken cancellationToken) + { + var relativeAddress = "index.json"; + var resourceUri = new Uri(storage.BaseAddress, relativeAddress); + var versions = GetVersions(await storage.LoadStringAsync(resourceUri, cancellationToken)); + + return new VersionsResult(relativeAddress, resourceUri, versions); + } + + private static HashSet GetVersions(string json) + { + var result = new HashSet(); + if (json != null) + { + JObject obj = JObject.Parse(json); + + JArray versions = obj["versions"] as JArray; + + if (versions != null) + { + foreach (JToken version in versions) + { + result.Add(NuGetVersion.Parse(version.ToString())); + } + } + } + return result; + } + + private StorageContent CreateContent(IEnumerable versions) + { + JObject obj = new JObject { { "versions", new JArray(versions) } }; + return new StringStorageContent(obj.ToString(), "application/json", "no-store"); + } + + private async Task SaveNupkgAsync(Stream nupkgStream, Storage storage, string id, string version, CancellationToken cancellationToken) + { + Uri nupkgUri = new Uri(storage.BaseAddress, GetRelativeAddressNupkg(id, version)); + var content = new StreamStorageContent( + nupkgStream, + DnxConstants.ApplicationOctetStreamContentType, + DnxConstants.DefaultCacheControl); + + await storage.SaveAsync(nupkgUri, content, cancellationToken); + + return nupkgUri; + } + + private async Task CopyNupkgAsync( + IStorage sourceStorage, + Storage destinationStorage, + string id, string version, CancellationToken cancellationToken) + { + var packageFileName = PackageUtility.GetPackageFileName(id, version); + var sourceUri = sourceStorage.ResolveUri(packageFileName); + var destinationRelativeUri = GetRelativeAddressNupkg(id, version); + var destinationUri = destinationStorage.ResolveUri(destinationRelativeUri); + + await sourceStorage.CopyAsync( + sourceUri, + destinationStorage, + destinationUri, + DnxConstants.RequiredBlobProperties, + cancellationToken); + + return destinationUri; + } + + private async Task CopyIconFromAzureStorageIfExistAsync( + IAzureStorage sourceStorage, + Storage destinationStorage, + string packageId, + string normalizedPackageVersion, + string iconFilename, + CancellationToken cancellationToken) + { + using (var packageStream = await GetPackageStreamAsync(sourceStorage, packageId, normalizedPackageVersion, cancellationToken)) + { + await CopyIconAsync( + packageStream, + iconFilename, + destinationStorage, + packageId, + normalizedPackageVersion, + cancellationToken); + } + } + + private async Task CopyIconFromNupkgStreamAsync( + Stream nupkgStream, + string iconFilename, + Storage destinationStorage, + string packageId, + string normalizedPackageVersion, + CancellationToken cancellationToken) + { + await CopyIconAsync( + nupkgStream, + iconFilename, + destinationStorage, + packageId, + normalizedPackageVersion, + cancellationToken); + } + + private async Task CopyIconAsync( + Stream packageStream, + string iconFilename, + Storage destinationStorage, + string packageId, + string normalizedPackageVersion, + CancellationToken cancellationToken) + { + _logger.LogInformation("Processing icon {IconFilename} for the package {PackageId} {PackageVersion}", + iconFilename, + packageId, + normalizedPackageVersion); + + var iconPath = PathUtility.StripLeadingDirectorySeparators(iconFilename); + + var destinationRelativeUri = GetRelativeAddressIcon(packageId, normalizedPackageVersion); + var destinationUri = destinationStorage.ResolveUri(destinationRelativeUri); + + await ExtractAndStoreIconAsync( + packageStream, + iconPath, + destinationStorage, + destinationUri, + cancellationToken, + packageId, + normalizedPackageVersion); + } + + private async Task ExtractAndStoreIconAsync( + Stream packageStream, + string iconPath, + Storage destinationStorage, + Uri destinationUri, + CancellationToken cancellationToken, + string packageId, + string normalizedPackageVersion) + { + using (var zipArchive = new ZipArchive(packageStream, ZipArchiveMode.Read, leaveOpen: true)) + { + var iconEntry = zipArchive.Entries.FirstOrDefault(e => e.FullName.Equals(iconPath, StringComparison.InvariantCultureIgnoreCase)); + if (iconEntry != null) + { + using (var iconStream = iconEntry.Open()) + { + _logger.LogInformation("Extracting icon to the destination storage {DestinationUri}", destinationUri); + // TODO: align the mime type determination with Gallery https://github.com/nuget/nugetgallery/issues/7061 + var iconContent = new StreamStorageContent(iconStream, string.Empty, DnxConstants.DefaultCacheControl); + await destinationStorage.SaveAsync(destinationUri, iconContent, cancellationToken); + _logger.LogInformation("Done"); + } + } + else + { + _telemetryService.TrackIconExtractionFailure(packageId, normalizedPackageVersion); + _logger.LogWarning("Zip archive entry {IconPath} does not exist", iconPath); + } + } + } + + private async Task GetPackageStreamAsync( + IAzureStorage sourceStorage, + string packageId, + string normalizedPackageVersion, + CancellationToken cancellationToken) + { + var packageFileName = PackageUtility.GetPackageFileName(packageId, normalizedPackageVersion); + var sourceUri = sourceStorage.ResolveUri(packageFileName); + var packageSourceBlob = await sourceStorage.GetCloudBlockBlobReferenceAsync(sourceUri); + return await packageSourceBlob.GetStreamAsync(cancellationToken); + } + + private async Task DeleteNuspecAsync(Storage storage, string id, string version, CancellationToken cancellationToken) + { + string relativeAddress = GetRelativeAddressNuspec(id, version); + Uri nuspecUri = new Uri(storage.BaseAddress, relativeAddress); + if (storage.Exists(relativeAddress)) + { + await storage.DeleteAsync(nuspecUri, cancellationToken); + } + } + + private async Task DeleteNupkgAsync(Storage storage, string id, string version, CancellationToken cancellationToken) + { + string relativeAddress = GetRelativeAddressNupkg(id, version); + Uri nupkgUri = new Uri(storage.BaseAddress, relativeAddress); + if (storage.Exists(relativeAddress)) + { + await storage.DeleteAsync(nupkgUri, cancellationToken); + } + } + + private async Task DeleteIconAsync(Storage storage, string id, string version, CancellationToken cancellationToken) + { + string relativeAddress = GetRelativeAddressIcon(id, version); + Uri iconUri = new Uri(storage.BaseAddress, relativeAddress); + if (storage.Exists(relativeAddress)) + { + await storage.DeleteAsync(iconUri, cancellationToken); + } + } + + private static string GetRelativeAddressNuspec(string id, string version) + { + return $"{NuGetVersion.Parse(version).ToNormalizedString()}/{id}.nuspec"; + } + + public static string GetRelativeAddressNupkg(string id, string version) + { + if (string.IsNullOrEmpty(id)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(id)); + } + + if (string.IsNullOrEmpty(version)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(version)); + } + + var normalizedVersion = NuGetVersion.Parse(version).ToNormalizedString(); + + return $"{normalizedVersion}/{id}.{normalizedVersion}.nupkg"; + } + + private static string GetRelativeAddressIcon(string id, string version) + { + var normalizedVersion = NuGetVersion.Parse(version).ToNormalizedString(); + + return $"{normalizedVersion}/icon"; + } + + private class VersionsResult + { + public VersionsResult(string relativeAddress, Uri resourceUri, HashSet versions) + { + RelativeAddress = relativeAddress; + ResourceUri = resourceUri; + Versions = versions; + } + + public string RelativeAddress { get; } + public Uri ResourceUri { get; } + public HashSet Versions { get; } + } + } +} \ No newline at end of file diff --git a/src/Catalog/DurableCursor.cs b/src/Catalog/DurableCursor.cs new file mode 100644 index 000000000..a0a2a611b --- /dev/null +++ b/src/Catalog/DurableCursor.cs @@ -0,0 +1,46 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog +{ + public class DurableCursor : ReadWriteCursor + { + Uri _address; + Storage _storage; + DateTime _defaultValue; + + public DurableCursor(Uri address, Storage storage, DateTime defaultValue) + { + _address = address; + _storage = storage; + _defaultValue = defaultValue; + } + + public override async Task SaveAsync(CancellationToken cancellationToken) + { + JObject obj = new JObject { { "value", Value.ToString("O") } }; + StorageContent content = new StringStorageContent(obj.ToString(), "application/json", "no-store"); + await _storage.SaveAsync(_address, content, cancellationToken); + } + + public override async Task LoadAsync(CancellationToken cancellationToken) + { + string json = await _storage.LoadStringAsync(_address, cancellationToken); + + if (json == null) + { + Value = _defaultValue; + return; + } + + JObject obj = JObject.Parse(json); + Value = obj["value"].ToObject(); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Extensions/DateTimeExtensions.cs b/src/Catalog/Extensions/DateTimeExtensions.cs new file mode 100644 index 000000000..0cf0a8e88 --- /dev/null +++ b/src/Catalog/Extensions/DateTimeExtensions.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace System +{ + public static class DateTimeExtensions + { + public static DateTime ForceUtc(this DateTime date) + { + if (date.Kind != DateTimeKind.Utc) + { + date = new DateTime(date.Ticks, DateTimeKind.Utc); + } + + return date; + } + } +} \ No newline at end of file diff --git a/src/Catalog/Extensions/DbDataReaderExtensions.cs b/src/Catalog/Extensions/DbDataReaderExtensions.cs new file mode 100644 index 000000000..cd58b7fd0 --- /dev/null +++ b/src/Catalog/Extensions/DbDataReaderExtensions.cs @@ -0,0 +1,55 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace System.Data.Common +{ + public static class DbDataReaderExtensions + { + public static int? ReadInt32OrNull(this DbDataReader dataReader, string columnName) + { + return ReadColumnOrNull(dataReader, columnName, (r, o) => r.GetInt32(o), (int?)null); + } + + public static string ReadStringOrNull(this DbDataReader dataReader, string columnName) + { + return ReadColumnOrNull(dataReader, columnName, (r, o) => r.GetString(o), nullValue: null); + } + + private static T ReadColumnOrNull(DbDataReader dataReader, string columnName, Func provideValue, T nullValue) + { + if (dataReader == null) + { + throw new ArgumentNullException(nameof(dataReader)); + } + + if (columnName == null) + { + throw new ArgumentNullException(nameof(columnName)); + } + + if (provideValue == null) + { + throw new ArgumentNullException(nameof(provideValue)); + } + + try + { + var ordinal = dataReader.GetOrdinal(columnName); + + if (!dataReader.IsDBNull(ordinal)) + { + return provideValue(dataReader, ordinal); + } + + return nullValue; + + } + catch (IndexOutOfRangeException) + { + // Thrown by DbDataReader.GetOrdinal(string) when the column does not exist. + // The exception can be swallowed as the intention of this method is to return null instead. + return nullValue; + } + } + } +} \ No newline at end of file diff --git a/src/Catalog/Extensions/IDataRecordExtensions.cs b/src/Catalog/Extensions/IDataRecordExtensions.cs new file mode 100644 index 000000000..acc86f9e2 --- /dev/null +++ b/src/Catalog/Extensions/IDataRecordExtensions.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace System.Data +{ + /// + /// Extension methods that make working with more convenient. + /// + public static class IDataRecordExtensions + { + public static DateTime ReadNullableUtcDateTime(this IDataRecord dataRecord, string columnName) + { + if (dataRecord == null) + { + throw new ArgumentNullException(nameof(dataRecord)); + } + + if (columnName == null) + { + throw new ArgumentNullException(nameof(columnName)); + } + + return (dataRecord[columnName] == DBNull.Value ? DateTime.MinValue : ReadDateTime(dataRecord, columnName)).ForceUtc(); + } + + public static DateTime ReadDateTime(this IDataRecord dataRecord, string columnName) + { + if (dataRecord == null) + { + throw new ArgumentNullException(nameof(dataRecord)); + } + + if (columnName == null) + { + throw new ArgumentNullException(nameof(columnName)); + } + + return dataRecord.GetDateTime(dataRecord.GetOrdinal(columnName)); + } + } +} \ No newline at end of file diff --git a/src/Catalog/FetchCatalogCommitsAsync.cs b/src/Catalog/FetchCatalogCommitsAsync.cs new file mode 100644 index 000000000..cd6164346 --- /dev/null +++ b/src/Catalog/FetchCatalogCommitsAsync.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog +{ + internal delegate Task> FetchCatalogCommitsAsync( + CollectorHttpClient client, + ReadCursor front, + ReadCursor back, + CancellationToken cancellationToken); +} \ No newline at end of file diff --git a/src/Catalog/FixPackageHashHandler.cs b/src/Catalog/FixPackageHashHandler.cs new file mode 100644 index 000000000..11f1cdeaa --- /dev/null +++ b/src/Catalog/FixPackageHashHandler.cs @@ -0,0 +1,74 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.WindowsAzure.Storage; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog +{ + /// + /// Adds the Content MD5 property to package blobs that are missing it. + /// + public class FixPackageHashHandler : IPackagesContainerHandler + { + private readonly HttpClient _httpClient; + private readonly ITelemetryService _telemetryService; + private readonly ILogger _logger; + + public FixPackageHashHandler( + HttpClient httpClient, + ITelemetryService telemetryService, + ILogger logger) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ProcessPackageAsync(CatalogIndexEntry packageEntry, ICloudBlockBlob blob) + { + await blob.FetchAttributesAsync(CancellationToken.None); + + // Skip the package if it has a Content MD5 hash + if (blob.ContentMD5 != null) + { + _telemetryService.TrackPackageAlreadyHasHash(packageEntry.Id, packageEntry.Version); + return; + } + + // Download the blob and calculate its hash. We use HttpClient to download blobs as Azure Blob Sotrage SDK + // occassionally hangs. See: https://github.com/Azure/azure-storage-net/issues/470 + string hash; + using (var hashAlgorithm = MD5.Create()) + using (var packageStream = await _httpClient.GetStreamAsync(blob.Uri)) + { + var hashBytes = hashAlgorithm.ComputeHash(packageStream); + + hash = Convert.ToBase64String(hashBytes); + } + + blob.ContentMD5 = hash; + + var condition = AccessCondition.GenerateIfMatchCondition(blob.ETag); + await blob.SetPropertiesAsync( + condition, + options: null, + operationContext: null); + + _telemetryService.TrackPackageHashFixed(packageEntry.Id, packageEntry.Version); + + _logger.LogWarning( + "Updated package {PackageId} {PackageVersion}, set hash to '{Hash}' using ETag {ETag}", + packageEntry.Id, + packageEntry.Version, + hash, + blob.ETag); + } + } +} diff --git a/src/Catalog/FlatContainerPackagePathProvider.cs b/src/Catalog/FlatContainerPackagePathProvider.cs new file mode 100644 index 000000000..b583694ac --- /dev/null +++ b/src/Catalog/FlatContainerPackagePathProvider.cs @@ -0,0 +1,69 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog +{ + public class FlatContainerPackagePathProvider + { + private readonly string _container; + + public FlatContainerPackagePathProvider(string container) + { + _container = container; + } + + public string GetPackagePath(string id, string version) + { + if (id == null) + { + throw new ArgumentNullException(nameof(id)); + } + + if (version == null) + { + throw new ArgumentNullException(nameof(version)); + } + + var idLowerCase = id.ToLowerInvariant(); + var versionLowerCase = NuGetVersionUtility.NormalizeVersion(version).ToLowerInvariant(); + var packageFileName = PackageUtility.GetPackageFileName(idLowerCase, versionLowerCase); + + return $"{_container}/{idLowerCase}/{versionLowerCase}/{packageFileName}"; + } + + public string GetIconPath(string id, string version) + { + return GetIconPath(id, version, normalize: true); + } + + public string GetIconPath(string id, string version, bool normalize) + { + if (id == null) + { + throw new ArgumentNullException(nameof(id)); + } + + if (version == null) + { + throw new ArgumentNullException(nameof(version)); + } + + var idLowerCase = id.ToLowerInvariant(); + + string versionLowerCase; + if (normalize) + { + versionLowerCase = NuGetVersionUtility.NormalizeVersion(version).ToLowerInvariant(); + } + else + { + versionLowerCase = version.ToLowerInvariant(); + } + + return $"{_container}/{idLowerCase}/{versionLowerCase}/icon"; + } + } +} diff --git a/src/Catalog/GetCatalogCommitItemKey.cs b/src/Catalog/GetCatalogCommitItemKey.cs new file mode 100644 index 000000000..d28bf0659 --- /dev/null +++ b/src/Catalog/GetCatalogCommitItemKey.cs @@ -0,0 +1,7 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog +{ + public delegate string GetCatalogCommitItemKey(CatalogCommitItem item); +} \ No newline at end of file diff --git a/src/Catalog/Helpers/AsyncExtensions.cs b/src/Catalog/Helpers/AsyncExtensions.cs new file mode 100644 index 000000000..c2e1b277b --- /dev/null +++ b/src/Catalog/Helpers/AsyncExtensions.cs @@ -0,0 +1,47 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + public static class AsyncExtensions + { + public static Task ForEachAsync(this IEnumerable enumerable, int maxDegreeOfParallelism, Func func) + { + if (enumerable == null) + { + throw new ArgumentNullException(nameof(enumerable)); + } + + if (maxDegreeOfParallelism < 1) + { + throw new ArgumentOutOfRangeException( + nameof(maxDegreeOfParallelism), + string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue)); + } + + if (func == null) + { + throw new ArgumentNullException(nameof(func)); + } + + return Task.WhenAll( + from partition in Partitioner.Create(enumerable).GetPartitions(maxDegreeOfParallelism) + select Task.Run(async delegate + { + using (partition) + { + while (partition.MoveNext()) + { + await func(partition.Current); + } + } + })); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/CatalogProperties.cs b/src/Catalog/Helpers/CatalogProperties.cs new file mode 100644 index 000000000..acef7ed86 --- /dev/null +++ b/src/Catalog/Helpers/CatalogProperties.cs @@ -0,0 +1,89 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + public sealed class CatalogProperties + { + public DateTime? LastCreated { get; } + public DateTime? LastDeleted { get; } + public DateTime? LastEdited { get; } + + public CatalogProperties(DateTime? lastCreated, DateTime? lastDeleted, DateTime? lastEdited) + { + LastCreated = lastCreated; + LastDeleted = lastDeleted; + LastEdited = lastEdited; + } + + /// + /// Asynchronously reads and returns top-level metadata from the catalog's index.json. + /// + /// The metadata values include "nuget:lastCreated", "nuget:lastDeleted", and "nuget:lastEdited", + /// which are the timestamps of the catalog cursor. + /// + /// + /// + /// A task that represents the asynchronous operation. + /// The task result () returns a . + /// Thrown if is null. + /// Thrown if + /// is cancelled. + public static async Task ReadAsync( + IStorage storage, + ITelemetryService telemetryService, + CancellationToken cancellationToken) + { + if (storage == null) + { + throw new ArgumentNullException(nameof(storage)); + } + + if (telemetryService == null) + { + throw new ArgumentNullException(nameof(telemetryService)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + DateTime? lastCreated = null; + DateTime? lastDeleted = null; + DateTime? lastEdited = null; + + var stopwatch = Stopwatch.StartNew(); + var indexUri = storage.ResolveUri("index.json"); + var json = await storage.LoadStringAsync(indexUri, cancellationToken); + + if (json != null) + { + var obj = JObject.Parse(json); + telemetryService.TrackCatalogIndexReadDuration(stopwatch.Elapsed, indexUri); + JToken token; + + if (obj.TryGetValue("nuget:lastCreated", out token)) + { + lastCreated = token.ToObject().ToUniversalTime(); + } + + if (obj.TryGetValue("nuget:lastDeleted", out token)) + { + lastDeleted = token.ToObject().ToUniversalTime(); + } + + if (obj.TryGetValue("nuget:lastEdited", out token)) + { + lastEdited = token.ToObject().ToUniversalTime(); + } + } + + return new CatalogProperties(lastCreated, lastDeleted, lastEdited); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/CatalogWriterHelper.cs b/src/Catalog/Helpers/CatalogWriterHelper.cs new file mode 100644 index 000000000..9ab1dece6 --- /dev/null +++ b/src/Catalog/Helpers/CatalogWriterHelper.cs @@ -0,0 +1,164 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + /// + /// Helper methods for writing to the catalog. + /// + public static class CatalogWriterHelper + { + /// + /// Asynchronously writes package metadata to the catalog. + /// + /// A package catalog item creator. + /// Packages to download metadata for. + /// Storage. + /// The catalog's last created datetime. + /// The catalog's last edited datetime. + /// The catalog's last deleted datetime. + /// The maximum degree of parallelism for package processing. + /// true to include created packages; otherwise, false. + /// true to update the created cursor from the last edited cursor; + /// otherwise, false. + /// A cancellation token. + /// A telemetry service. + /// A logger. + /// A task that represents the asynchronous operation. + /// The task result () returns the latest + /// that was processed. + public static async Task WritePackageDetailsToCatalogAsync( + IPackageCatalogItemCreator packageCatalogItemCreator, + SortedList> packages, + IStorage storage, + DateTime lastCreated, + DateTime lastEdited, + DateTime lastDeleted, + int maxDegreeOfParallelism, + bool? createdPackages, + bool updateCreatedFromEdited, + CancellationToken cancellationToken, + ITelemetryService telemetryService, + ILogger logger) + { + if (packageCatalogItemCreator == null) + { + throw new ArgumentNullException(nameof(packageCatalogItemCreator)); + } + + if (packages == null) + { + throw new ArgumentNullException(nameof(packages)); + } + + if (storage == null) + { + throw new ArgumentNullException(nameof(storage)); + } + + if (maxDegreeOfParallelism < 1) + { + throw new ArgumentOutOfRangeException( + nameof(maxDegreeOfParallelism), + string.Format(Strings.ArgumentOutOfRange, 1, int.MaxValue)); + } + + if (telemetryService == null) + { + throw new ArgumentNullException(nameof(telemetryService)); + } + + if (logger == null) + { + throw new ArgumentNullException(nameof(logger)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var writer = new AppendOnlyCatalogWriter(storage, telemetryService, Constants.MaxPageSize); + + var lastDate = DetermineLastDate(lastCreated, lastEdited, createdPackages); + + if (packages.Count == 0) + { + return lastDate; + } + + // Flatten the sorted list. + var workItems = packages.SelectMany( + pair => pair.Value.Select( + details => new PackageWorkItem(pair.Key, details))) + .ToArray(); + + await workItems.ForEachAsync(maxDegreeOfParallelism, async workItem => + { + workItem.PackageCatalogItem = await packageCatalogItemCreator.CreateAsync( + workItem.FeedPackageDetails, + workItem.Timestamp, + cancellationToken); + }); + + lastDate = packages.Last().Key; + + // AppendOnlyCatalogWriter.Add(...) is not thread-safe, so add them all at once on one thread. + foreach (var workItem in workItems.Where(workItem => workItem.PackageCatalogItem != null)) + { + writer.Add(workItem.PackageCatalogItem); + + logger?.LogInformation("Add metadata from: {PackageDetailsContentUri}", workItem.FeedPackageDetails.ContentUri); + } + + if (createdPackages.HasValue) + { + lastEdited = !createdPackages.Value ? lastDate : lastEdited; + + if (updateCreatedFromEdited) + { + lastCreated = lastEdited; + } + else + { + lastCreated = createdPackages.Value ? lastDate : lastCreated; + } + } + + var commitMetadata = PackageCatalog.CreateCommitMetadata(writer.RootUri, new CommitMetadata(lastCreated, lastEdited, lastDeleted)); + + await writer.Commit(commitMetadata, cancellationToken); + + logger?.LogInformation("COMMIT metadata to catalog."); + + return lastDate; + } + + private static DateTime DetermineLastDate(DateTime lastCreated, DateTime lastEdited, bool? createdPackages) + { + if (!createdPackages.HasValue) + { + return DateTime.MinValue; + } + return createdPackages.Value ? lastCreated : lastEdited; + } + + private sealed class PackageWorkItem + { + internal DateTime Timestamp { get; } + internal FeedPackageDetails FeedPackageDetails { get; } + internal PackageCatalogItem PackageCatalogItem { get; set; } + + internal PackageWorkItem(DateTime timestamp, FeedPackageDetails feedPackageDetails) + { + Timestamp = timestamp; + FeedPackageDetails = feedPackageDetails; + } + } + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/Db2CatalogCursor.cs b/src/Catalog/Helpers/Db2CatalogCursor.cs new file mode 100644 index 000000000..a2704472b --- /dev/null +++ b/src/Catalog/Helpers/Db2CatalogCursor.cs @@ -0,0 +1,31 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Data.SqlTypes; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + public sealed class Db2CatalogCursor + { + private Db2CatalogCursor(string columnName, DateTime cursorValue, int top) + { + ColumnName = columnName ?? throw new ArgumentNullException(nameof(columnName)); + CursorValue = cursorValue < SqlDateTime.MinValue.Value ? SqlDateTime.MinValue.Value : cursorValue; + + if (top <= 0) + { + throw new ArgumentOutOfRangeException("Argument value must be a positive non-zero integer.", nameof(top)); + } + + Top = top; + } + + public string ColumnName { get; } + public DateTime CursorValue { get; } + public int Top { get; } + + public static Db2CatalogCursor ByCreated(DateTime since, int top) => new Db2CatalogCursor(Db2CatalogProjectionColumnNames.Created, since, top); + public static Db2CatalogCursor ByLastEdited(DateTime since, int top) => new Db2CatalogCursor(Db2CatalogProjectionColumnNames.LastEdited, since, top); + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/Db2CatalogProjection.cs b/src/Catalog/Helpers/Db2CatalogProjection.cs new file mode 100644 index 000000000..42521c5b8 --- /dev/null +++ b/src/Catalog/Helpers/Db2CatalogProjection.cs @@ -0,0 +1,134 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Linq; +using System.Threading.Tasks; +using NuGet.Protocol.Plugins; +using NuGet.Services.Entities; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + /// + /// Utility class to project s retrieved by db2catalog + /// into a format that is consumable by the catalog writer. + /// + public class Db2CatalogProjection + { + public const string AlternatePackageVersionWildCard = "*"; + + private readonly PackageContentUriBuilder _packageContentUriBuilder; + + public Db2CatalogProjection(PackageContentUriBuilder packageContentUriBuilder) + { + _packageContentUriBuilder = packageContentUriBuilder ?? throw new ArgumentNullException(nameof(packageContentUriBuilder)); + } + + public string ReadPackageVersionKeyFromDataReader(DbDataReader dataReader) => + dataReader[Db2CatalogProjectionColumnNames.Key].ToString(); + + /// + /// Note that this method will read details from current and end by reading next, closing reader (to communicate state) when end reached. + /// + public FeedPackageDetails ReadFeedPackageDetailsFromDataReader(DbDataReader dataReader) + { + if (dataReader == null) + { + throw new ArgumentNullException(nameof(dataReader)); + } + + var packageId = dataReader[Db2CatalogProjectionColumnNames.PackageId].ToString(); + var normalizedPackageVersion = dataReader[Db2CatalogProjectionColumnNames.NormalizedVersion].ToString(); + var fullPackageVersion = dataReader[Db2CatalogProjectionColumnNames.FullVersion].ToString(); + var listed = dataReader.GetBoolean(dataReader.GetOrdinal(Db2CatalogProjectionColumnNames.Listed)); + var hideLicenseReport = dataReader.GetBoolean(dataReader.GetOrdinal(Db2CatalogProjectionColumnNames.HideLicenseReport)); + + var packageContentUri = _packageContentUriBuilder.Build(packageId, normalizedPackageVersion); + var deprecationInfo = ReadDeprecationInfoFromDataReader(dataReader); + + return new FeedPackageDetails( + packageContentUri, + dataReader.ReadDateTime(Db2CatalogProjectionColumnNames.Created).ForceUtc(), + dataReader.ReadNullableUtcDateTime(Db2CatalogProjectionColumnNames.LastEdited), + listed ? dataReader.ReadDateTime(Db2CatalogProjectionColumnNames.Published).ForceUtc() : Constants.UnpublishedDate, + packageId, + normalizedPackageVersion, + fullPackageVersion, + hideLicenseReport ? null : dataReader[Db2CatalogProjectionColumnNames.LicenseNames]?.ToString(), + hideLicenseReport ? null : dataReader[Db2CatalogProjectionColumnNames.LicenseReportUrl]?.ToString(), + deprecationInfo, + dataReader.GetBoolean(dataReader.GetOrdinal(Db2CatalogProjectionColumnNames.RequiresLicenseAcceptance))); + } + + public PackageVulnerabilityItem ReadPackageVulnerabilityFromDataReader(DbDataReader dataReader) + { + var gitHubDatabaseKey = dataReader[Db2CatalogProjectionColumnNames.VulnerabilityGitHubDatabaseKey].ToString(); + var advisoryUrl = dataReader[Db2CatalogProjectionColumnNames.VulnerabilityAdvisoryUrl].ToString(); + var severity = dataReader[Db2CatalogProjectionColumnNames.VulnerabilitySeverity].ToString(); + + if (string.IsNullOrEmpty(gitHubDatabaseKey) || string.IsNullOrEmpty(advisoryUrl) || string.IsNullOrEmpty(severity)) + { + return null; + } + + return new PackageVulnerabilityItem(gitHubDatabaseKey: gitHubDatabaseKey, advisoryUrl: advisoryUrl, severity: severity); + } + + public PackageDeprecationItem ReadDeprecationInfoFromDataReader(DbDataReader dataReader) + { + if (dataReader == null) + { + throw new ArgumentNullException(nameof(dataReader)); + } + + var deprecationReasons = new List(); + var deprecationStatusValue = dataReader.ReadInt32OrNull(Db2CatalogProjectionColumnNames.DeprecationStatus); + + if (!deprecationStatusValue.HasValue) + { + return null; + } + + var deprecationStatus = (PackageDeprecationStatus)deprecationStatusValue.Value; + + foreach (var deprecationStatusFlag in Enum.GetValues(typeof(PackageDeprecationStatus)).Cast()) + { + if (deprecationStatusFlag == PackageDeprecationStatus.NotDeprecated) + { + continue; + } + + if (deprecationStatus.HasFlag(deprecationStatusFlag)) + { + deprecationReasons.Add(deprecationStatusFlag.ToString()); + } + } + + var alternatePackageId = dataReader.ReadStringOrNull(Db2CatalogProjectionColumnNames.AlternatePackageId); + string alternatePackageVersion = null; + if (alternatePackageId != null) + { + alternatePackageVersion = dataReader.ReadStringOrNull(Db2CatalogProjectionColumnNames.AlternatePackageVersion); + + if (alternatePackageVersion == null) + { + alternatePackageVersion = AlternatePackageVersionWildCard; + } + else + { + // The alternate package version range is defined by a minimum version (lower-bound), or higher (no upper-bound). + alternatePackageVersion = $"[{alternatePackageVersion}, )"; + } + } + + return new PackageDeprecationItem( + deprecationReasons, + dataReader.ReadStringOrNull(Db2CatalogProjectionColumnNames.DeprecationMessage), + alternatePackageId, + alternatePackageVersion); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/Db2CatalogProjectionColumnNames.cs b/src/Catalog/Helpers/Db2CatalogProjectionColumnNames.cs new file mode 100644 index 000000000..f321f11ca --- /dev/null +++ b/src/Catalog/Helpers/Db2CatalogProjectionColumnNames.cs @@ -0,0 +1,31 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + /// + /// Defines the column names for db2catalog SQL queries. + /// + public static class Db2CatalogProjectionColumnNames + { + public const string Key = "Key"; + public const string PackageId = "Id"; + public const string NormalizedVersion = "NormalizedVersion"; + public const string FullVersion = "Version"; + public const string Listed = "Listed"; + public const string HideLicenseReport = "HideLicenseReport"; + public const string Created = "Created"; + public const string LastEdited = "LastEdited"; + public const string Published = "Published"; + public const string LicenseNames = "LicenseNames"; + public const string LicenseReportUrl = "LicenseReportUrl"; + public const string AlternatePackageId = "AlternatePackageId"; + public const string AlternatePackageVersion = "AlternatePackageVersion"; + public const string DeprecationStatus = "DeprecationStatus"; + public const string DeprecationMessage = "DeprecationMessage"; + public const string RequiresLicenseAcceptance = "RequiresLicenseAcceptance"; + public const string VulnerabilityGitHubDatabaseKey = "VulnerabilityGitHubDatabaseKey"; + public const string VulnerabilityAdvisoryUrl = "VulnerabilityAdvisoryUrl"; + public const string VulnerabilitySeverity = "VulnerabilitySeverity"; + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/DeletionAuditEntry.cs b/src/Catalog/Helpers/DeletionAuditEntry.cs new file mode 100644 index 000000000..0d70e264e --- /dev/null +++ b/src/Catalog/Helpers/DeletionAuditEntry.cs @@ -0,0 +1,261 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + /// + /// Represents the information in the audit entry for a deletion. + /// + public class DeletionAuditEntry + { + /// + /// Over time, we have had multiple file names for audit records for deletes. + /// + public readonly static IList FileNameSuffixes = new List + { + "-Deleted.audit.v1.json", + "-deleted.audit.v1.json", + "-softdeleted.audit.v1.json", + "-softdelete.audit.v1.json", + "-delete.audit.v1.json" + }; + + /// + /// Asynchronously creates a from a + /// and an . + /// + /// The through which + /// can be accessed. + /// A to the record to build a + /// from. + /// A cancellation token. + /// A logger. + /// A task that represents the asynchronous operation. + /// The task result () returns a + /// instance. + public static async Task CreateAsync( + IStorage auditingStorage, + Uri uri, + CancellationToken cancellationToken, + ILogger logger) + { + try + { + return new DeletionAuditEntry(uri, await auditingStorage.LoadStringAsync(uri, cancellationToken)); + } + catch (JsonReaderException) + { + logger.LogWarning("Audit record at {AuditRecordUri} contains invalid JSON.", uri); + } + catch (NullReferenceException) + { + logger.LogWarning("Audit record at {AuditRecordUri} does not contain required JSON properties to perform a package delete.", uri); + } + catch (ArgumentException) + { + logger.LogWarning("Audit record at {AuditRecordUri} has no contents.", uri); + } + + return null; + } + + private DeletionAuditEntry(Uri uri, string contents) + { + if (string.IsNullOrEmpty(contents)) + { + throw new ArgumentException($"{nameof(contents)} must not be null or empty!", nameof(contents)); + } + + Uri = uri; + Record = JObject.Parse(contents); + InitValues(); + } + + [JsonConstructor] + public DeletionAuditEntry() + { + } + + /// + /// Constructor for testing. + /// + public DeletionAuditEntry(Uri uri, JObject record, string id, string version, DateTime? timestamp) + { + Uri = uri; + Record = record; + PackageId = id; + PackageVersion = version; + TimestampUtc = timestamp; + } + + /// + /// The for the audit entry. + /// + [JsonProperty("uri")] + public Uri Uri { get; private set; } + + /// + /// The entire contents of the audit entry file. + /// + [JsonProperty("record")] + public JObject Record { get; private set; } + + /// + /// The id of the package being audited. + /// + [JsonProperty("id")] + public string PackageId { get; private set; } + + /// + /// The version of the package being audited. + /// + [JsonProperty("version")] + public string PackageVersion { get; private set; } + + /// + /// The the package was deleted. + /// + [JsonProperty("timestamp")] + public DateTime? TimestampUtc { get; private set; } + + private const string RecordPart = "Record"; + private const string ActorPart = "Actor"; + + private JObject GetPart(string partName) + { + return (JObject)Record?.GetValue(partName, StringComparison.OrdinalIgnoreCase); + } + + private void InitValues() + { + PackageId = GetPart(RecordPart).GetValue("Id", StringComparison.OrdinalIgnoreCase).ToString(); + PackageVersion = GetPart(RecordPart).GetValue("Version", StringComparison.OrdinalIgnoreCase).ToString(); + TimestampUtc = + GetPart(ActorPart).GetValue("TimestampUtc", StringComparison.OrdinalIgnoreCase).Value(); + } + + /// + /// Fetches s. + /// + /// The to fetch audit records from. + /// A that can be used to cancel the task. + /// If specified, will only fetch s that represent operations on this package. + /// If specified, will only fetch s that are newer than this (non-inclusive). + /// If specified, will only fetch s that are older than this (non-inclusive). + /// An to log messages to. + /// An containing the relevant s. + public static Task> GetAsync( + StorageFactory auditingStorageFactory, + CancellationToken cancellationToken, + PackageIdentity package = null, + DateTime? minTime = null, + DateTime? maxTime = null, + ILogger logger = null) + { + Storage storage = auditingStorageFactory.Create(package != null ? GetAuditRecordPrefixFromPackage(package) : null); + return GetAsync(storage, cancellationToken, minTime, maxTime, logger); + } + + /// + /// Asynchronously fetches s. + /// + /// The to fetch audit records from. + /// A that can be used to cancel + /// the task. + /// If specified, will only fetch s that are newer than + /// this (non-inclusive). + /// If specified, will only fetch s that are older than + /// this (non-inclusive). + /// An to log messages to. + /// A task that represents the asynchronous operation. + /// The task result () returns an + /// containing the relevant + /// s. + public static async Task> GetAsync( + IStorage auditingStorage, + CancellationToken cancellationToken, + DateTime? minTime = null, + DateTime? maxTime = null, + ILogger logger = null) + { + Func filterAuditRecord = (record) => + { + if (!IsPackageDelete(record)) + { + return false; + } + + // We can't do anything if the last modified time is not available. + if (record.LastModifiedUtc == null) + { + logger?.LogWarning("Could not get date for filename in filterAuditRecord. Uri: {AuditRecordUri}", record.Uri); + return false; + } + + var recordTimestamp = record.LastModifiedUtc.Value; + if (minTime != null && recordTimestamp < minTime.Value) + { + return false; + } + + if (maxTime != null && recordTimestamp > maxTime.Value) + { + return false; + } + + return true; + }; + + // Get all audit blobs (based on their filename which starts with a date that can be parsed). + /// Filter on the and fields provided. + var auditRecords = + (await auditingStorage.ListAsync(cancellationToken)).Where(filterAuditRecord); + + return + (await Task.WhenAll( + auditRecords.Select(record => DeletionAuditEntry.CreateAsync(auditingStorage, record.Uri, cancellationToken, logger)))) + // Filter out null records. + .Where(entry => entry?.Record != null); + } + + /// + /// Returns the prefix of the audit record, which contains the id and version of the package being audited. + /// + private static string GetAuditRecordPrefix(Uri uri) + { + var parts = uri.PathAndQuery.Split('/'); + return string.Join("/", parts.Where(p => !string.IsNullOrEmpty(p)).ToList().GetRange(0, parts.Length - 2).ToArray()); + } + + private static string GetAuditRecordPrefixFromPackage(PackageIdentity package) + { + return $"{package.Id.ToLowerInvariant()}/{package.Version.ToNormalizedString().ToLowerInvariant()}"; + } + + /// + /// Returns the file name of the audit record, which contains the the record was made as well as the type of record it is. + /// + private static string GetAuditRecordFileName(Uri uri) + { + var parts = uri.PathAndQuery.Split('/'); + return parts.Length > 0 ? parts[parts.Length - 1] : null; + } + + private static bool IsPackageDelete(StorageListItem auditRecord) + { + var fileName = GetAuditRecordFileName(auditRecord.Uri); + return FileNameSuffixes.Any(suffix => fileName.EndsWith(suffix)); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/FeedHelpers.cs b/src/Catalog/Helpers/FeedHelpers.cs new file mode 100644 index 000000000..28be296c2 --- /dev/null +++ b/src/Catalog/Helpers/FeedHelpers.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + /// + /// Helper methods for accessing and parsing the gallery's V2 feed. + /// + public static class FeedHelpers + { + /// + /// Creates an HttpClient for reading the feed. + /// + public static HttpClient CreateHttpClient(Func handlerFunc) + { + var handler = (handlerFunc != null) ? handlerFunc() : new WebRequestHandler { AllowPipelining = true }; + return new HttpClient(handler); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/FeedPackageDetails.cs b/src/Catalog/Helpers/FeedPackageDetails.cs new file mode 100644 index 000000000..e165d72a5 --- /dev/null +++ b/src/Catalog/Helpers/FeedPackageDetails.cs @@ -0,0 +1,82 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + public sealed class FeedPackageDetails + { + public Uri ContentUri { get; } + public DateTime CreatedDate { get; } + public DateTime LastEditedDate { get; } + public DateTime PublishedDate { get; } + public string PackageId { get; } + public string PackageNormalizedVersion { get; } + public string PackageFullVersion { get; } + public string LicenseNames { get; } + public string LicenseReportUrl { get; } + public bool RequiresLicenseAcceptance { get; } + public PackageDeprecationItem DeprecationInfo { get; } + public IList VulnerabilityInfo { get; private set; } + + public bool HasDeprecationInfo => DeprecationInfo != null; + + public FeedPackageDetails( + Uri contentUri, + DateTime createdDate, + DateTime lastEditedDate, + DateTime publishedDate, + string packageId, + string packageNormalizedVersion, + string packageFullVersion) + : this( + contentUri, + createdDate, + lastEditedDate, + publishedDate, + packageId, + packageNormalizedVersion, + packageFullVersion, + licenseNames: null, + licenseReportUrl: null, + deprecationInfo: null, + requiresLicenseAcceptance: false) + { + } + + public FeedPackageDetails( + Uri contentUri, + DateTime createdDate, + DateTime lastEditedDate, + DateTime publishedDate, + string packageId, + string packageNormalizedVersion, + string packageFullVersion, + string licenseNames, + string licenseReportUrl, + PackageDeprecationItem deprecationInfo, + bool requiresLicenseAcceptance) + { + ContentUri = contentUri; + CreatedDate = createdDate; + LastEditedDate = lastEditedDate; + PublishedDate = publishedDate; + PackageId = packageId; + PackageNormalizedVersion = packageNormalizedVersion; + PackageFullVersion = packageFullVersion; + LicenseNames = licenseNames; + LicenseReportUrl = licenseReportUrl; + DeprecationInfo = deprecationInfo; + RequiresLicenseAcceptance = requiresLicenseAcceptance; + } + + public void AddVulnerability(PackageVulnerabilityItem vulnerability) + { + VulnerabilityInfo = VulnerabilityInfo ?? new List(); + VulnerabilityInfo.Add(vulnerability); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/FeedPackageIdentity.cs b/src/Catalog/Helpers/FeedPackageIdentity.cs new file mode 100644 index 000000000..fac5cd5f4 --- /dev/null +++ b/src/Catalog/Helpers/FeedPackageIdentity.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; +using NuGet.Packaging.Core; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + public class FeedPackageIdentity : IEquatable + { + [JsonProperty("id")] + public string Id { get; private set; } + + [JsonProperty("version")] + public string Version { get; private set; } + + [JsonConstructor] + public FeedPackageIdentity(string id, string version) + { + Id = id; + Version = version; + } + + public FeedPackageIdentity(PackageIdentity package) + { + Id = package.Id; + Version = package.Version.ToFullString(); + } + + public bool Equals(FeedPackageIdentity other) + { + return Id.ToLowerInvariant() == other.Id.ToLowerInvariant() && Version.ToLowerInvariant() == other.Version.ToLowerInvariant(); + } + + public override int GetHashCode() + { + return Tuple.Create(Id.ToLowerInvariant(), Version.ToLowerInvariant()).GetHashCode(); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/GalleryDatabaseQueryService.cs b/src/Catalog/Helpers/GalleryDatabaseQueryService.cs new file mode 100644 index 000000000..b82898ae1 --- /dev/null +++ b/src/Catalog/Helpers/GalleryDatabaseQueryService.cs @@ -0,0 +1,287 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Data.SqlClient; +using System.Linq; +using System.Threading.Tasks; +using NuGet.Services.Entities; +using NuGet.Services.Sql; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + /// + /// Utility class for all SQL queries invoked by Db2Catalog. + /// + public class GalleryDatabaseQueryService : IGalleryDatabaseQueryService + { + private const string CursorParameterName = "Cursor"; + private const string PackageIdParameterName = "PackageId"; + private const string PackageVersionParameterName = "PackageVersion"; + + // insertions are: + // {0} - inner SELECT type + // {1} - additional WHERE conditions and ORDER BY if needed for inner SELECT's TOP n + // {2} - outer ORDER BY if needed + private static readonly string Db2CatalogSqlSubQuery = @"SELECT + P_EXT.*, + PV.[GitHubDatabaseKey] AS '" + Db2CatalogProjectionColumnNames.VulnerabilityGitHubDatabaseKey + @"', + PV.[AdvisoryUrl] AS '" + Db2CatalogProjectionColumnNames.VulnerabilityAdvisoryUrl + @"', + PV.[Severity] AS '" + Db2CatalogProjectionColumnNames.VulnerabilitySeverity + @"' + FROM + ({0} P.[Key], + PR.[Id], + P.[NormalizedVersion], + P.[Version], + P.[Created], + P.[LastEdited], + P.[Published], + P.[Listed], + P.[HideLicenseReport], + P.[LicenseNames], + P.[LicenseReportUrl], + P.[RequiresLicenseAcceptance], + PD.[Status] AS '" + Db2CatalogProjectionColumnNames.DeprecationStatus + @"', + APR.[Id] AS '" + Db2CatalogProjectionColumnNames.AlternatePackageId + @"', + AP.[NormalizedVersion] AS '" + Db2CatalogProjectionColumnNames.AlternatePackageVersion + @"', + PD.[CustomMessage] AS '" + Db2CatalogProjectionColumnNames.DeprecationMessage + @"' + FROM [dbo].[Packages] AS P + INNER JOIN [dbo].[PackageRegistrations] AS PR ON P.[PackageRegistrationKey] = PR.[Key] + LEFT JOIN [dbo].[PackageDeprecations] AS PD ON PD.[PackageKey] = P.[Key] + LEFT JOIN [dbo].[Packages] AS AP ON AP.[Key] = PD.[AlternatePackageKey] + LEFT JOIN [dbo].[PackageRegistrations] AS APR ON APR.[Key] = ISNULL(AP.[PackageRegistrationKey], PD.[AlternatePackageRegistrationKey]) + WHERE P.[PackageStatusKey] = 0 + {1} + ) AS P_EXT + LEFT JOIN [dbo].[VulnerablePackageVersionRangePackages] AS VPVRP ON VPVRP.[Package_Key] = P_EXT.[Key] + LEFT JOIN [dbo].[VulnerablePackageVersionRanges] AS VPVR ON VPVR.[Key] = VPVRP.[VulnerablePackageVersionRange_Key] + LEFT JOIN [dbo].[PackageVulnerabilities] AS PV ON PV.[Key] = VPVR.[VulnerabilityKey] + {2}"; + + private readonly ISqlConnectionFactory _connectionFactory; + private readonly Db2CatalogProjection _db2catalogProjection; + private readonly ITelemetryService _telemetryService; + private readonly int _commandTimeout; + + public GalleryDatabaseQueryService( + ISqlConnectionFactory connectionFactory, + PackageContentUriBuilder packageContentUriBuilder, + ITelemetryService telemetryService, + int commandTimeout) + { + _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _db2catalogProjection = new Db2CatalogProjection(packageContentUriBuilder); + _commandTimeout = commandTimeout; + } + + public Task>> GetPackagesCreatedSince(DateTime since, int top) + { + return GetPackagesInOrder( + package => package.CreatedDate, + Db2CatalogCursor.ByCreated(since, top)); + } + + public Task>> GetPackagesEditedSince(DateTime since, int top) + { + return GetPackagesInOrder( + package => package.LastEditedDate, + Db2CatalogCursor.ByLastEdited(since, top)); + } + + public async Task GetPackageOrNull(string id, string version) + { + if (id == null) + { + throw new ArgumentNullException(nameof(id)); + } + + if (version == null) + { + throw new ArgumentNullException(nameof(version)); + } + + var packages = new List(); + var packageQuery = BuildGetPackageSqlQuery(); + + using (var sqlConnection = await _connectionFactory.OpenAsync()) + { + using (var packagesCommand = new SqlCommand(packageQuery, sqlConnection) + { + CommandTimeout = _commandTimeout + }) + { + packagesCommand.Parameters.AddWithValue(PackageIdParameterName, id); + packagesCommand.Parameters.AddWithValue(PackageVersionParameterName, version); + + using (_telemetryService.TrackGetPackageQueryDuration(id, version)) + { + return (await ReadPackagesAsync(packagesCommand)).SingleOrDefault(); + } + } + } + } + + /// + /// Returns a from the gallery database. + /// + /// The field to sort the on. + private async Task>> GetPackagesInOrder( + Func keyDateFunc, + Db2CatalogCursor cursor) + { + var allPackages = await GetPackages(cursor); + + return OrderPackagesByKeyDate(allPackages, keyDateFunc); + } + + /// + /// Returns a of packages. + /// + /// The field to sort the on. + internal static SortedList> OrderPackagesByKeyDate( + IReadOnlyCollection packages, + Func keyDateFunc) + { + var result = new SortedList>(); + + foreach (var package in packages) + { + var packageKeyDate = keyDateFunc(package); + if (!result.TryGetValue(packageKeyDate, out IList packagesWithSameKeyDate)) + { + packagesWithSameKeyDate = new List(); + result.Add(packageKeyDate, packagesWithSameKeyDate); + } + + packagesWithSameKeyDate.Add(package); + } + + var packagesCount = 0; + var filteredResult = new SortedList>(); + foreach (var keyDate in result.Keys) + { + if (result.TryGetValue(keyDate, out IList packagesForKeyDate)) + { + if (packagesCount > 0 && packagesCount + packagesForKeyDate.Count > Constants.MaxPageSize) + { + break; + } + + packagesCount += packagesForKeyDate.Count; + filteredResult.Add(keyDate, packagesForKeyDate); + } + } + + return filteredResult; + } + + /// + /// Builds the SQL query string for db2catalog. + /// + /// The to be used. + /// The SQL query string for the db2catalog job, build from the the provided . + internal static string BuildDb2CatalogSqlQuery(Db2CatalogCursor cursor) + { + // We need to provide an inner ORDER BY to support the TOP clause + return string.Format(Db2CatalogSqlSubQuery, + $@"SELECT TOP {cursor.Top} WITH TIES ", + $@"AND P.[{cursor.ColumnName}] > @{CursorParameterName} + ORDER BY P.[{cursor.ColumnName}]", + $@"ORDER BY P_EXT.[{cursor.ColumnName}], P_EXT.[{Db2CatalogProjectionColumnNames.Key}]"); + } + + /// + /// Builds the parameterized SQL query for retrieving package details given an id and version. + /// + internal static string BuildGetPackageSqlQuery() + { + return string.Format(Db2CatalogSqlSubQuery, + "SELECT ", + "AND PR.[Id] = @PackageId AND P.[NormalizedVersion] = @PackageVersion", + ""); + } + + /// + /// Asynchronously gets a from the gallery database. + /// + /// Defines the cursor to be used. + /// A task that represents the asynchronous operation. + /// The task result () returns an + /// . + private async Task> GetPackages(Db2CatalogCursor cursor) + { + using (var sqlConnection = await _connectionFactory.OpenAsync()) + { + return await GetPackageDetailsAsync(sqlConnection, cursor); + } + } + + private async Task> GetPackageDetailsAsync( + SqlConnection sqlConnection, + Db2CatalogCursor cursor) + { + var packageQuery = BuildDb2CatalogSqlQuery(cursor); + + using (var packagesCommand = new SqlCommand(packageQuery, sqlConnection) + { + CommandTimeout = _commandTimeout + }) + { + packagesCommand.Parameters.AddWithValue(CursorParameterName, cursor.CursorValue); + + using (_telemetryService.TrackGetPackageDetailsQueryDuration(cursor)) + { + return await ReadPackagesAsync(packagesCommand); + } + } + } + + private async Task> ReadPackagesAsync(SqlCommand packagesCommand) + { + var packages = new List(); + + using (var packagesReader = await packagesCommand.ExecuteReaderAsync()) + { + // query has been ordered by package/version key, so we can use control break logic here to read it efficiently + // - we break on key change to find the end of a group of rows for which we must accumulate vulnerability data + if (await packagesReader.ReadAsync()) // priming read + { + var key = _db2catalogProjection.ReadPackageVersionKeyFromDataReader(packagesReader); + var readEnded = false; + do + { + var package = _db2catalogProjection.ReadFeedPackageDetailsFromDataReader(packagesReader); + packages.Add(package); + + // loop through all rows with the same key, so we cover all vulnerabilities for package/version + var thisKey = key; + do + { + var vulnerability = _db2catalogProjection.ReadPackageVulnerabilityFromDataReader(packagesReader); + if (vulnerability != null) + { + package.AddVulnerability(vulnerability); + } + + // read next, prime for control break + if (await packagesReader.ReadAsync()) + { + key = _db2catalogProjection.ReadPackageVersionKeyFromDataReader(packagesReader); + } + else + { + readEnded = true; + } + } while (!readEnded && key == thisKey); + } while (!readEnded); + } + } + + return packages; + } + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/GraphLoading.cs b/src/Catalog/Helpers/GraphLoading.cs new file mode 100644 index 000000000..9f5441fd8 --- /dev/null +++ b/src/Catalog/Helpers/GraphLoading.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading.Tasks; +using VDS.RDF; + +namespace NuGet.Services.Metadata.Catalog +{ + public static class GraphLoading + { + static async Task Load(Uri uri) + { + HttpClient client = new HttpClient(); + string json = await client.GetStringAsync(uri); + return Utils.CreateGraph(json); + } + + public static async Task Load(Uri root, IDictionary rules) + { + ISet resourceList = new HashSet(); + resourceList.Add(root); + + IGraph graph = await Load(root); + + bool dirty = true; + + while (dirty) + { + dirty = false; + + foreach (KeyValuePair rule in rules) + { + foreach (Triple t1 in graph.GetTriplesWithPredicateObject( + graph.CreateUriNode(new Uri("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")), + graph.CreateUriNode(new Uri(rule.Key)))) + { + foreach (Triple t2 in graph.GetTriplesWithSubjectPredicate( + t1.Subject, + graph.CreateUriNode(new Uri(rule.Value)))) + { + Uri next = ((IUriNode)t2.Object).Uri; + + if (!resourceList.Contains(next)) + { + IGraph nextGraph = await Load(next); + graph.Merge(nextGraph, true); + + resourceList.Add(next); + + dirty = true; + } + } + } + } + } + + return graph; + } + } +} diff --git a/src/Catalog/Helpers/GraphSplitting.cs b/src/Catalog/Helpers/GraphSplitting.cs new file mode 100644 index 000000000..93289f9d3 --- /dev/null +++ b/src/Catalog/Helpers/GraphSplitting.cs @@ -0,0 +1,197 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using VDS.RDF; + +namespace NuGet.Services.Metadata.Catalog +{ + public static class GraphSplitting + { + public static Uri GetPackageRegistrationUri(IGraph graph) + { + return ((IUriNode)graph.GetTriplesWithPredicateObject( + graph.CreateUriNode(new Uri("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")), + graph.CreateUriNode(new Uri("http://schema.nuget.org/schema#PackageRegistration"))) + .First().Subject).Uri; + } + public static IList GetResources(IGraph graph) + { + IList resources = new List(); + + INode rdfType = graph.CreateUriNode(new Uri("http://www.w3.org/1999/02/22-rdf-syntax-ns#type")); + + Uri[] types = new Uri[] + { + //new Uri("http://schema.nuget.org/schema#PackageRegistration"), + new Uri("http://schema.nuget.org/schema#PackageList") + }; + + foreach (Uri type in types) + { + foreach (Triple triple in graph.GetTriplesWithPredicateObject(rdfType, graph.CreateUriNode(type))) + { + resources.Add(((IUriNode)triple.Subject).Uri); + } + } + + return resources; + } + + public static IGraph ReplaceResourceUris(IGraph original, IDictionary replacements) + { + IGraph modified = new Graph(); + foreach (Triple triple in original.Triples) + { + Uri subjectUri; + if (!replacements.TryGetValue(triple.Subject.ToString(), out subjectUri)) + { + subjectUri = ((IUriNode)triple.Subject).Uri; + } + + INode subjectNode = modified.CreateUriNode(subjectUri); + INode predicateNode = triple.Predicate.CopyNode(modified); + + INode objectNode; + if (triple.Object is IUriNode) + { + Uri objectUri; + if (!replacements.TryGetValue(triple.Object.ToString(), out objectUri)) + { + objectUri = ((IUriNode)triple.Object).Uri; + } + objectNode = modified.CreateUriNode(objectUri); + } + else + { + objectNode = triple.Object.CopyNode(modified); + } + + modified.Assert(subjectNode, predicateNode, objectNode); + } + + return modified; + } + public static void Collect(IGraph source, INode subject, IGraph destination, ISet exclude) + { + foreach (Triple triple in source.GetTriplesWithSubject(subject)) + { + destination.Assert(triple.CopyTriple(destination)); + + if (triple.Object is IUriNode && !exclude.Contains(((IUriNode)triple.Object).Uri.ToString())) + { + Collect(source, triple.Object, destination, exclude); + } + } + } + + static Uri RebaseUri(Uri nodeUri, Uri sourceUri, Uri destinationUri) + { + if (nodeUri == sourceUri && nodeUri.ToString() != sourceUri.ToString()) + { + return new Uri(destinationUri.ToString() + nodeUri.Fragment); + } + return nodeUri; + } + public static void Rebase(IGraph source, IGraph destination, Uri sourceUri, Uri destinationUri) + { + Uri modifiedDestinationUri = new Uri(destinationUri.ToString().Replace('#', '/')); + + foreach (Triple triple in source.Triples) + { + Uri subjectUri; + if (triple.Subject.ToString() == destinationUri.ToString()) + { + subjectUri = modifiedDestinationUri; + } + else + { + subjectUri = RebaseUri(((IUriNode)triple.Subject).Uri, sourceUri, modifiedDestinationUri); + } + + INode subjectNode = destination.CreateUriNode(subjectUri); + INode predicateNode = triple.Predicate.CopyNode(destination); + + INode objectNode; + if (triple.Object is IUriNode) + { + Uri objectUri = RebaseUri(((IUriNode)triple.Object).Uri, sourceUri, modifiedDestinationUri); + objectNode = destination.CreateUriNode(objectUri); + } + else + { + objectNode = triple.Object.CopyNode(destination); + } + + destination.Assert(subjectNode, predicateNode, objectNode); + } + } + + static IDictionary CreateReplacements(Uri originalUri, IList resources) + { + IDictionary replacements = new Dictionary(); + + foreach (Uri resource in resources) + { + if (resource == originalUri) + { + string oldUri = resource.ToString(); + string newUri = oldUri.Replace(".json#", "/"); + + if (!newUri.EndsWith(".json")) + { + newUri += ".json"; + } + + replacements.Add(oldUri, new Uri(newUri)); + } + } + + return replacements; + } + + public static IDictionary Split(Uri originalUri, IGraph originalGraph) + { + IList resources = GraphSplitting.GetResources(originalGraph); + + IDictionary replacements = CreateReplacements(originalUri, resources); + + ISet exclude = new HashSet(); + exclude.Add(originalUri.ToString()); + + IGraph modified = GraphSplitting.ReplaceResourceUris(originalGraph, replacements); + + IDictionary graphs = new Dictionary(); + + IGraph parent = new Graph(); + + foreach (Triple triple in modified.Triples) + { + triple.CopyTriple(parent); + parent.Assert(triple); + } + + foreach (Uri uri in replacements.Values) + { + INode subject = modified.CreateUriNode(uri); + + IGraph cutGraph = new Graph(); + GraphSplitting.Collect(modified, subject, cutGraph, exclude); + + foreach (Triple triple in cutGraph.Triples) + { + triple.CopyTriple(parent); + parent.Retract(triple); + } + + IGraph rebasedGraph = new Graph(); + GraphSplitting.Rebase(cutGraph, rebasedGraph, originalUri, uri); + + graphs.Add(uri, rebasedGraph); + } + + graphs.Add(originalUri, parent); + + return graphs; + } + } +} diff --git a/src/Catalog/Helpers/IGalleryDatabaseQueryService.cs b/src/Catalog/Helpers/IGalleryDatabaseQueryService.cs new file mode 100644 index 000000000..cd204a348 --- /dev/null +++ b/src/Catalog/Helpers/IGalleryDatabaseQueryService.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + public interface IGalleryDatabaseQueryService + { + Task GetPackageOrNull(string id, string version); + Task>> GetPackagesCreatedSince(DateTime since, int top); + Task>> GetPackagesEditedSince(DateTime since, int top); + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/JsonSort.cs b/src/Catalog/Helpers/JsonSort.cs new file mode 100644 index 000000000..0e147bebf --- /dev/null +++ b/src/Catalog/Helpers/JsonSort.cs @@ -0,0 +1,110 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + /// + /// This is a hacky attempt at organizing compacted json into a more visually appealing form. + /// + public class JsonSort : IComparer + { + /// + /// Order the json so arrays are at the bottom and single properties are at the top + /// + public static JObject OrderJson(JObject json) + { + JObject ordered = new JObject(); + + var children = json.Children().ToList(); + + children.Sort(new JsonSort()); + + foreach (var child in children) + { + ordered.Add(child); + } + + return ordered; + } + + public int Compare(JToken x, JToken y) + { + JProperty xProp = x as JProperty; + JProperty yProp = y as JProperty; + + if (xProp != null && yProp == null) + { + return -1; + } + + if (xProp == null && yProp != null) + { + return 1; + } + + if (xProp != null && yProp != null) + { + if (xProp.Name.Equals("@id")) + { + return -1; + } + + if (yProp.Name.Equals("@id")) + { + return 1; + } + + if (xProp.Name.Equals("@type")) + { + return -1; + } + + if (yProp.Name.Equals("@type")) + { + return 1; + } + + if (xProp.Name.Equals("@context")) + { + return 1; + } + + if (yProp.Name.Equals("@context")) + { + return -1; + } + + JArray xValArray = xProp.Value as JArray; + JArray yValArray = yProp.Value as JArray; + + if (xValArray == null && yValArray != null) + { + return -1; + } + + if (xValArray != null && yValArray == null) + { + return 1; + } + + if (xProp.Name.StartsWith("@") && !yProp.Name.StartsWith("@")) + { + return 1; + } + + if (!xProp.Name.StartsWith("@") && yProp.Name.StartsWith("@")) + { + return -1; + } + + return StringComparer.OrdinalIgnoreCase.Compare(xProp.Name, yProp.Name); + } + + return 0; + } + } +} diff --git a/src/Catalog/Helpers/LicenseHelper.cs b/src/Catalog/Helpers/LicenseHelper.cs new file mode 100644 index 000000000..04566c93c --- /dev/null +++ b/src/Catalog/Helpers/LicenseHelper.cs @@ -0,0 +1,30 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog +{ + public static class LicenseHelper + { + /// + /// Generate the license url given package info and gallery base url. + /// + /// package Id + /// package version + /// url of gallery base address + /// The url of license in gallery + public static string GetGalleryLicenseUrl(string packageId, string packageVersion, Uri galleryBaseAddress) + { + if (galleryBaseAddress == null || string.IsNullOrWhiteSpace(packageId) || string.IsNullOrWhiteSpace(packageVersion)) + { + return null; + } + + var uriBuilder = new UriBuilder(galleryBaseAddress); + uriBuilder.Path = string.Join("/", new string[] { "packages", packageId, packageVersion, "license" }); + + return uriBuilder.Uri.AbsoluteUri; + } + } +} diff --git a/src/Catalog/Helpers/NuGetVersionUtility.cs b/src/Catalog/Helpers/NuGetVersionUtility.cs new file mode 100644 index 000000000..70722a886 --- /dev/null +++ b/src/Catalog/Helpers/NuGetVersionUtility.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Versioning; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + public static class NuGetVersionUtility + { + public static string NormalizeVersion(string version) + { + NuGetVersion parsedVersion; + if (!NuGetVersion.TryParse(version, out parsedVersion)) + { + return version; + } + + return parsedVersion.ToNormalizedString(); + } + + public static string NormalizeVersionRange(string versionRange, string defaultValue) + { + VersionRange parsedVersionRange; + if (!VersionRange.TryParse(versionRange, out parsedVersionRange)) + { + return defaultValue; + } + + return parsedVersionRange.ToNormalizedString(); + } + + public static string GetFullVersionString(string version) + { + NuGetVersion parsedVersion; + if (!NuGetVersion.TryParse(version, out parsedVersion)) + { + return version; + } + + return parsedVersion.ToFullString(); + } + } +} diff --git a/src/Catalog/Helpers/PackageContentUriBuilder.cs b/src/Catalog/Helpers/PackageContentUriBuilder.cs new file mode 100644 index 000000000..b82e372ec --- /dev/null +++ b/src/Catalog/Helpers/PackageContentUriBuilder.cs @@ -0,0 +1,56 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + /// + /// Utility class to validate package content URL format strings. + /// + public class PackageContentUriBuilder + { + public const string IdLowerPlaceholderString = "{id-lower}"; + public const string VersionLowerPlaceholderString = "{version-lower}"; + + private const string NuPkgExtension = ".nupkg"; + private readonly string _packageContentUrlFormat; + + public PackageContentUriBuilder(string packageContentUrlFormat) + { + _packageContentUrlFormat = packageContentUrlFormat ?? throw new ArgumentNullException(nameof(packageContentUrlFormat)); + + if (!_packageContentUrlFormat.Contains(IdLowerPlaceholderString) || !_packageContentUrlFormat.Contains(VersionLowerPlaceholderString)) + { + throw new ArgumentException( + $"The package content URL format must contain the following placeholders to be valid: {IdLowerPlaceholderString}, {VersionLowerPlaceholderString}. " + + "(e.g. https://storageaccountname.blob.core.windows.net/packages/{id-lower}.{version-lower}.nupkg)", + nameof(packageContentUrlFormat)); + } + else if (!_packageContentUrlFormat.EndsWith(NuPkgExtension)) + { + throw new ArgumentException( + $"The package content URL format must point to files with the {NuPkgExtension} extension.", + nameof(packageContentUrlFormat)); + } + } + + public Uri Build(string packageId, string normalizedPackageVersion) + { + if (packageId == null) + { + throw new ArgumentNullException(nameof(packageId)); + } + + if (normalizedPackageVersion == null) + { + throw new ArgumentNullException(nameof(normalizedPackageVersion)); + } + + return new Uri( + _packageContentUrlFormat + .Replace(IdLowerPlaceholderString, packageId.ToLowerInvariant()) + .Replace(VersionLowerPlaceholderString, normalizedPackageVersion.ToLowerInvariant())); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/PackageUtility.cs b/src/Catalog/Helpers/PackageUtility.cs new file mode 100644 index 000000000..22cf8dd8c --- /dev/null +++ b/src/Catalog/Helpers/PackageUtility.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + public static class PackageUtility + { + public static string GetPackageFileName(string packageId, string packageVersion) + { + if (string.IsNullOrEmpty(packageId)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageId)); + } + + if (string.IsNullOrEmpty(packageVersion)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(packageVersion)); + } + + return $"{packageId}.{packageVersion}.nupkg"; + } + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/ParallelAsync.cs b/src/Catalog/Helpers/ParallelAsync.cs new file mode 100644 index 000000000..6dd851bfb --- /dev/null +++ b/src/Catalog/Helpers/ParallelAsync.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Net; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + public static class ParallelAsync + { + /// + /// Creates a number of tasks specified by using and then runs them in parallel. + /// + /// Creates each task to run. + /// The number of tasks to create. Defaults to + /// A task that completes when all tasks have completed. + public static Task Repeat(Func taskFactory, int? degreeOfParallelism = null) + { + return Task.WhenAll( + Enumerable + .Repeat(taskFactory, degreeOfParallelism ?? ServicePointManager.DefaultConnectionLimit) + .Select(f => f())); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/Retry.cs b/src/Catalog/Helpers/Retry.cs new file mode 100644 index 000000000..614a5b76d --- /dev/null +++ b/src/Catalog/Helpers/Retry.cs @@ -0,0 +1,84 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + /// + /// Can (and probably should) be replaced with Polly library if the project is updated to target .netfx 4.7.2. + /// In current state Polly pulls a ton of System.* dependencies which we previously didn't have. + /// + public class Retry + { + /// + /// Retries async operation if it throws with delays between attempts. + /// + /// Operation to try. + /// Exception predicate. If it returns false, the exception will propagate to the caller. + /// Max number of attempts to make. + /// Delay after the first failure. + /// Delay increment for subsequent attempts. + public static async Task IncrementalAsync( + Func runLogicAsync, + Func shouldRetryOnException, + int maxRetries, + TimeSpan initialWaitInterval, + TimeSpan waitIncrement) + { + for (int currentRetry = 0; currentRetry < maxRetries; ++currentRetry) + { + try + { + await runLogicAsync(); + return; + } + catch (Exception e) when (currentRetry < maxRetries - 1 && shouldRetryOnException(e)) + { + await Task.Delay(initialWaitInterval + TimeSpan.FromSeconds(waitIncrement.TotalSeconds * currentRetry)); + } + } + } + + /// + /// Retries async operation if it throws or returns certain result with delays between attempts. + /// + /// Attempted operation result type. + /// Operation to try. + /// Exception predicate. If it returns false, the exception will propagate to the caller. + /// Result predicate. If returns true, the result will be discarded and operation retried. + /// Max number of attempts to make. + /// Delay after the first failure. + /// Delay increment for subsequent attempts. + /// The result of () call if predicate fails. + /// default() if suceeded for all attempts. + public static async Task IncrementalAsync( + Func> runLogicAsync, + Func shouldRetryOnException, + Func shouldRetry, + int maxRetries, + TimeSpan initialWaitInterval, + TimeSpan waitIncrement) + { + var result = default(TResult); + for (int currentRetry = 0; currentRetry < maxRetries; ++currentRetry) + { + try + { + result = await runLogicAsync(); + if (!shouldRetry(result)) + { + return result; + } + } + catch (Exception e) when (currentRetry < maxRetries - 1 && shouldRetryOnException(e)) + { + await Task.Delay(initialWaitInterval + TimeSpan.FromSeconds(waitIncrement.TotalSeconds * currentRetry)); + } + } + + return result; + } + } +} diff --git a/src/Catalog/Helpers/UriUtils.cs b/src/Catalog/Helpers/UriUtils.cs new file mode 100644 index 000000000..f27a368b2 --- /dev/null +++ b/src/Catalog/Helpers/UriUtils.cs @@ -0,0 +1,113 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using NuGet.Versioning; + +namespace NuGet.Services.Metadata.Catalog.Helpers +{ + public static class UriUtils + { + private const char QueryStartCharacter = '?'; + private const char BridgingCharacter = '&'; + + private const string Filter = "$filter="; + private const string NonhijackableFilter = "true"; + private const string And = "%20and%20"; + + private const string OrderBy = "$orderby="; + private const string NonhijackableOrderBy = "Version"; + + private const string GetSpecificPackageFormatRegExp = + @"Packages\(Id='(?[^']*)',Version='(?[^']*)'\)"; + + /// + /// The format of a nonhijacked "Packages(Id='...',Version='...')" request. + /// + /// + /// Note that we are using "normalized version" here. + /// This is because a hijacked request does a comparison on the normalized version, but the nonhijacked request does not. + /// Additionally, we must add the "semVerLevel=2.0.0" or else this will not work for SemVer2 packages. + /// + private static string GetSpecificPackageNonhijackable = + $"Packages?{Filter}{NonhijackableFilter}{And}" + + "Id eq '{0}'" + $"{And}" + "NormalizedVersion eq '{1}'&semVerLevel=2.0.0"; + + private static IEnumerable HijackableEndpoints = new List + { + "/Packages", + "/Search", + "/FindPackagesById" + }; + + public static Uri GetNonhijackableUri(Uri originalUri) + { + var nonhijackableUri = originalUri; + + var originalUriString = originalUri.ToString(); + string nonhijackableUriString = null; + + // Modify the request uri so that it will not be hijacked by the search service. + + // This can be done in two ways: + /// 1 - convert the query into a "Packages" query with filter that cannot be hijacked (). + /// 2 - specify an orderby that cannot be hijacked (). + if (originalUriString.Contains(OrderBy)) + { + // If there is an orderby on the request, simply replace the orderby with a nonhijackable orderby. + var orderByStartIndex = originalUriString.IndexOf(OrderBy); + + /// Find the start of the next parameter () or the end of the query. + var orderByEndIndex = originalUriString.IndexOf(BridgingCharacter, orderByStartIndex); + if (orderByEndIndex == -1) + { + orderByEndIndex = originalUriString.Length; + } + + // Replace the entire orderby with a nonhijackable orderby. + var orderByExpression = originalUriString.Substring(orderByStartIndex, orderByEndIndex - orderByStartIndex); + nonhijackableUriString = originalUriString.Replace(orderByExpression, OrderBy + NonhijackableOrderBy); + } + else + { + // If this is a Packages(Id='...',Version='...') request, rewrite it as a request to Packages() with a filter and add the expression. + // Note that Packages() returns a feed and Packages(Id='...',Version='...') returns an entry, but this is fine because the client reads both the same. + var getSpecificPackageMatch = Regex.Match(originalUriString, GetSpecificPackageFormatRegExp); + if (getSpecificPackageMatch.Success) + { + nonhijackableUriString = + originalUriString.Substring(0, getSpecificPackageMatch.Index) + + string.Format( + GetSpecificPackageNonhijackable, + getSpecificPackageMatch.Groups["Id"].Value, + NuGetVersion.Parse(getSpecificPackageMatch.Groups["Version"].Value).ToNormalizedString()); + } + else + { + // If this is a request to a hijackable endpoint without an orderby, add the orderby to the request. + if (HijackableEndpoints.Any(endpoint => originalUriString.Contains(endpoint))) + { + var bridgingCharacter = BridgingCharacter; + + if (!originalUriString.Contains(QueryStartCharacter)) + { + bridgingCharacter = QueryStartCharacter; + } + + nonhijackableUriString = $"{originalUri}{bridgingCharacter}{OrderBy}{NonhijackableOrderBy}"; + } + } + } + + if (nonhijackableUriString != null) + { + nonhijackableUri = new Uri(nonhijackableUriString); + } + + return nonhijackableUri; + } + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/Utils.cs b/src/Catalog/Helpers/Utils.cs new file mode 100644 index 000000000..a26ad14a5 --- /dev/null +++ b/src/Catalog/Helpers/Utils.cs @@ -0,0 +1,447 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; +using System.Xml; +using System.Xml.Linq; +using System.Xml.Xsl; +using JsonLD.Core; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.JsonLDIntegration; +using VDS.RDF; +using VDS.RDF.Parsing; + +namespace NuGet.Services.Metadata.Catalog +{ + public static class Utils + { + private const string XslTransformNuSpec = "xslt.nuspec.xslt"; + private const string XslTransformNormalizeNuSpecNamespace = "xslt.normalizeNuspecNamespace.xslt"; + + private static readonly Lazy XslTransformNuSpecCache = new Lazy(() => SafeLoadXslTransform(XslTransformNuSpec)); + private static readonly Lazy XslTransformNormalizeNuSpecNamespaceCache = new Lazy(() => SafeLoadXslTransform(XslTransformNormalizeNuSpecNamespace)); + + private static readonly char[] TagTrimChars = { ',', ' ', '\t', '|', ';' }; + + public static string[] SplitTags(string original) + { + var fields = original + .Split(TagTrimChars) + .Select(w => w.Trim(TagTrimChars)) + .Where(w => w.Length > 0) + .ToArray(); + + return fields; + } + + public static Stream GetResourceStream(string resourceName) + { + if (string.IsNullOrEmpty(resourceName)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(resourceName)); + } + + var assembly = Assembly.GetExecutingAssembly(); + + string name = assembly.GetName().Name; + + return assembly.GetManifestResourceStream($"{name}.{resourceName}"); + } + + public static IGraph CreateNuspecGraph(XDocument nuspec, string baseAddress, bool normalizeXml = false) + { + XsltArgumentList arguments = new XsltArgumentList(); + arguments.AddParam("base", "", baseAddress); + arguments.AddParam("extension", "", ".json"); + + arguments.AddExtensionObject("urn:helper", new XsltHelper()); + + nuspec = SafeXmlTransform(nuspec.CreateReader(), XslTransformNormalizeNuSpecNamespaceCache.Value); + var rdfxml = SafeXmlTransform(nuspec.CreateReader(), XslTransformNuSpecCache.Value, arguments); + + var doc = SafeCreateXmlDocument(rdfxml.CreateReader()); + if (normalizeXml) + { + NormalizeXml(doc); + } + + RdfXmlParser rdfXmlParser = new RdfXmlParser(); + IGraph graph = new Graph(); + rdfXmlParser.Load(graph, doc); + + return graph; + } + + private static void NormalizeXml(XmlNode xmlNode) + { + if (xmlNode.Attributes != null) + { + foreach (XmlAttribute attribute in xmlNode.Attributes) + { + attribute.Value = attribute.Value.Normalize(NormalizationForm.FormC); + } + } + + if (xmlNode.Value != null) + { + xmlNode.Value = xmlNode.Value.Normalize(NormalizationForm.FormC); + return; + } + + foreach (XmlNode childNode in xmlNode.ChildNodes) + { + NormalizeXml(childNode); + } + } + + internal static XmlDocument SafeCreateXmlDocument(XmlReader reader = null) + { + // CodeAnalysis / XmlDocument: set the resolver to null or instance + var xmlDoc = new XmlDocument(); + xmlDoc.XmlResolver = null; + + if (reader != null) + { + xmlDoc.Load(reader); + } + + return xmlDoc; + } + + private static XDocument SafeXmlTransform(XmlReader reader, XslCompiledTransform transform, XsltArgumentList arguments = null) + { + XDocument result = new XDocument(); + using (XmlWriter writer = result.CreateWriter()) + { + if (arguments == null) + { + arguments = new XsltArgumentList(); + } + + // CodeAnalysis / XslCompiledTransform.Transform: set resolver property to null or instance + transform.Transform(reader, arguments, writer, documentResolver: null); + } + return result; + } + + private static XslCompiledTransform SafeLoadXslTransform(string resourceName) + { + var transform = new XslCompiledTransform(); + + // CodeAnalysis / XmlReader.Create: provide settings instance and set resolver property to null or instance + var settings = new XmlReaderSettings(); + settings.XmlResolver = null; + + var reader = XmlReader.Create(new StreamReader(GetResourceStream(resourceName)), settings); + + // CodeAnalysis / XslCompiledTransform.Load: specify default settings or set resolver property to null or instance + transform.Load(reader, XsltSettings.Default, stylesheetResolver: null); + return transform; + } + + public static XDocument GetNuspec(ZipArchive package) + { + if (package == null) { return null; } + + foreach (ZipArchiveEntry part in package.Entries) + { + if (part.FullName.EndsWith(".nuspec") && part.FullName.IndexOf('/') == -1) + { + XDocument nuspec = XDocument.Load(part.Open()); + return nuspec; + } + } + return null; + } + + public static JToken CreateJson(IGraph graph, JToken frame = null) + { + System.IO.StringWriter writer = new System.IO.StringWriter(); + IRdfWriter rdfWriter = new JsonLdWriter(); + rdfWriter.Save(graph, writer); + writer.Flush(); + + if (frame == null) + { + return JToken.Parse(writer.ToString()); + } + else + { + JToken flattened = JToken.Parse(writer.ToString()); + JObject framed = JsonLdProcessor.Frame(flattened, frame, new JsonLdOptions()); + JObject compacted = JsonLdProcessor.Compact(framed, frame["@context"], new JsonLdOptions()); + + return JsonSort.OrderJson(compacted); + } + } + + public static string CreateArrangedJson(IGraph graph, JToken frame = null) + { + System.IO.StringWriter writer = new System.IO.StringWriter(); + IRdfWriter rdfWriter = new JsonLdWriter(); + rdfWriter.Save(graph, writer); + writer.Flush(); + + if (frame == null) + { + return writer.ToString(); + } + else + { + JToken flattened = JToken.Parse(writer.ToString()); + JObject framed = JsonLdProcessor.Frame(flattened, frame, new JsonLdOptions()); + JObject compacted = JsonLdProcessor.Compact(framed, frame["@context"], new JsonLdOptions()); + + var arranged = JsonSort.OrderJson(compacted); + + return arranged.ToString(); + } + } + + public static IGraph CreateGraph(Uri resourceUri, string json) + { + if (json == null) + { + return null; + } + + try + { + JToken compacted = JToken.Parse(json); + return CreateGraph(compacted, readOnly: false); + } + catch (JsonException e) + { + Trace.TraceError("Exception: failed to parse {0} {1}", resourceUri, e); + throw; + } + } + + public static IGraph CreateGraph(JToken compacted, bool readOnly) + { + JToken flattened = JsonLdProcessor.Flatten(compacted, new JsonLdOptions()); + + IRdfReader rdfReader = new JsonLdReader(); + IGraph graph = new Graph(); + rdfReader.Load(graph, new StringReader(flattened.ToString(Newtonsoft.Json.Formatting.None, new Newtonsoft.Json.JsonConverter[0]))); + + if (readOnly) + { + graph = new ReadOnlyGraph(graph); + } + + return graph; + } + + public static bool IsCatalogNode(INode sourceNode, IGraph source) + { + Triple rootTriple = source.GetTriplesWithSubjectObject(sourceNode, source.CreateUriNode(Schema.DataTypes.CatalogRoot)).FirstOrDefault(); + Triple pageTriple = source.GetTriplesWithSubjectObject(sourceNode, source.CreateUriNode(Schema.DataTypes.CatalogPage)).FirstOrDefault(); + + return (rootTriple != null || pageTriple != null); + } + + public static void CopyCatalogContentGraph(INode sourceNode, IGraph source, IGraph target) + { + if (IsCatalogNode(sourceNode, source)) + { + return; + } + + foreach (Triple triple in source.GetTriplesWithSubject(sourceNode)) + { + if (target.Assert(triple.CopyTriple(target)) && triple.Object is IUriNode) + { + CopyCatalogContentGraph(triple.Object, source, target); + } + } + } + + public static Uri Expand(JToken context, string term) + { + if (term.StartsWith("http:", StringComparison.OrdinalIgnoreCase)) + { + return new Uri(term); + } + + int indexOf = term.IndexOf(':'); + if (indexOf > 0) + { + string ns = term.Substring(0, indexOf); + return new Uri(context[ns].ToString() + term.Substring(indexOf + 1)); + } + + return new Uri(context["@vocab"] + term); + } + + // where the property exists on the graph being merged in remove it from the existing graph + public static void RemoveExistingProperties(IGraph existingGraph, IGraph graphToMerge, Uri[] properties) + { + foreach (Uri property in properties) + { + foreach (Triple t1 in graphToMerge.GetTriplesWithPredicate(graphToMerge.CreateUriNode(property))) + { + INode subject = t1.Subject.CopyNode(existingGraph); + INode predicate = t1.Predicate.CopyNode(existingGraph); + + IList retractList = new List(existingGraph.GetTriplesWithSubjectPredicate(subject, predicate)); + foreach (Triple t2 in retractList) + { + existingGraph.Retract(t2); + } + } + } + } + + public static string GenerateHash(Stream stream) + { + stream.Seek(0, SeekOrigin.Begin); + + using (var hashAlgorithm = HashAlgorithm.Create(Constants.Sha512)) + { + return Convert.ToBase64String(hashAlgorithm.ComputeHash(stream)); + } + } + + public static IEnumerable GetEntries(ZipArchive package) + { + IList result = new List(); + + foreach (ZipArchiveEntry entry in package.Entries) + { + if (entry.FullName.EndsWith("/.rels", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (entry.FullName.EndsWith("[Content_Types].xml", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (entry.FullName.EndsWith(".psmdcp", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + result.Add(new PackageEntry(entry)); + } + + return result; + } + + public static NupkgMetadata GetNupkgMetadata(Stream stream, string packageHash) + { + if (stream == null) + { + throw new ArgumentNullException(nameof(stream)); + } + + var packageSize = stream.Length; + + packageHash = packageHash ?? GenerateHash(stream); + + stream.Seek(0, SeekOrigin.Begin); + + using (var package = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true)) + { + var nuspec = GetNuspec(package); + + if (nuspec == null) + { + throw new InvalidDataException("Unable to find nuspec"); + } + + var entries = GetEntries(package); + + return new NupkgMetadata(nuspec, entries, packageSize, packageHash); + } + } + + public static PackageCatalogItem CreateCatalogItem( + string origin, + Stream stream, + DateTime createdDate, + DateTime? lastEditedDate = null, + DateTime? publishedDate = null, + string licenseNames = null, + string licenseReportUrl = null, + string packageHash = null, + PackageDeprecationItem deprecationItem = null, + IList vulnerabilities = null) + { + try + { + NupkgMetadata nupkgMetadata = GetNupkgMetadata(stream, packageHash); + return new PackageCatalogItem( + nupkgMetadata, + createdDate, + lastEditedDate, + publishedDate, + deprecation: deprecationItem, + vulnerabilities: vulnerabilities); + } + catch (InvalidDataException e) + { + Trace.TraceError("Exception: {0} {1} {2}", origin, e.GetType().Name, e); + return null; + } + catch (Exception e) + { + throw new Exception(string.Format("Exception processsing {0}", origin), e); + } + } + + public static void TraceException(Exception e) + { + if (e is AggregateException) + { + foreach (Exception ex in ((AggregateException)e).InnerExceptions) + { + TraceException(ex); + } + } + else + { + Trace.TraceError("{0} {1}", e.GetType().Name, e.Message); + Trace.TraceError("{0}", e.StackTrace); + + if (e.InnerException != null) + { + TraceException(e.InnerException); + } + } + } + + internal static T Deserialize(JObject jObject, string propertyName) + { + if (jObject == null) + { + throw new ArgumentNullException(nameof(jObject)); + } + + if (string.IsNullOrEmpty(propertyName)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(propertyName)); + } + + if (!jObject.TryGetValue(propertyName, out var value) || value == null) + { + throw new ArgumentException(string.Format(CultureInfo.InvariantCulture, Strings.PropertyRequired, propertyName)); + } + + return value.ToObject(); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Helpers/XsltHelper.cs b/src/Catalog/Helpers/XsltHelper.cs new file mode 100644 index 000000000..819016491 --- /dev/null +++ b/src/Catalog/Helpers/XsltHelper.cs @@ -0,0 +1,68 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Xml; +using System.Xml.XPath; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Versioning; + +namespace NuGet.Services.Metadata.Catalog +{ + public class XsltHelper + { + /// + /// Default to an empty string if the dependency version range is invalid or missing. This is meant to be a + /// predictable signal to the client that they need to handle this invalid version case. The official NuGet + /// client treats this as a dependency of any version. + /// + private static readonly string DefaultVersionRange = string.Empty; + + public XPathNavigator Split(string original) + { + var fields = Utils.SplitTags(original); + + XmlDocument xmlDoc = Utils.SafeCreateXmlDocument(); + XmlElement root = xmlDoc.CreateElement("list"); + xmlDoc.AppendChild(root); + + foreach (string s in fields) + { + XmlElement element = xmlDoc.CreateElement("item"); + element.InnerText = s; + root.AppendChild(element); + } + + return xmlDoc.CreateNavigator(); + } + + public string LowerCase(string original) + { + return original.ToLowerInvariant(); + } + + public string NormalizeVersion(string original) + { + return NuGetVersionUtility.NormalizeVersion(original); + } + + public string GetFullVersionString(string original) + { + return NuGetVersionUtility.GetFullVersionString(original); + } + + public string NormalizeVersionRange(string original) + { + return NuGetVersionUtility.NormalizeVersionRange(original, DefaultVersionRange); + } + + public string IsPrerelease(string original) + { + NuGetVersion nugetVersion; + if (NuGetVersion.TryParse(original, out nugetVersion)) + { + return nugetVersion.IsPrerelease ? "true" : "false"; + } + return "true"; + } + } +} diff --git a/src/Catalog/HttpReadCursor.cs b/src/Catalog/HttpReadCursor.cs new file mode 100644 index 000000000..b7202d8b6 --- /dev/null +++ b/src/Catalog/HttpReadCursor.cs @@ -0,0 +1,70 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog +{ + public class HttpReadCursor : ReadCursor + { + private readonly Uri _address; + private readonly DateTime? _defaultValue; + private readonly Func _handlerFunc; + + public HttpReadCursor(Uri address, DateTime defaultValue, Func handlerFunc = null) + { + _address = address; + _defaultValue = defaultValue; + _handlerFunc = handlerFunc; + } + + public HttpReadCursor(Uri address, Func handlerFunc = null) + { + _address = address; + _defaultValue = null; + _handlerFunc = handlerFunc; + } + + public override async Task LoadAsync(CancellationToken cancellationToken) + { + await Retry.IncrementalAsync( + async () => + { + HttpMessageHandler handler = (_handlerFunc != null) ? _handlerFunc() : new WebRequestHandler { AllowPipelining = true }; + + using (HttpClient client = new HttpClient(handler)) + using (HttpResponseMessage response = await client.GetAsync(_address, cancellationToken)) + { + Trace.TraceInformation("HttpReadCursor.Load {0} {1}", response.StatusCode, _address.AbsoluteUri); + + if (_defaultValue != null && response.StatusCode == HttpStatusCode.NotFound) + { + Value = _defaultValue.Value; + } + else + { + response.EnsureSuccessStatusCode(); + + string json = await response.Content.ReadAsStringAsync(); + + JObject obj = JObject.Parse(json); + Value = obj["value"].ToObject(); + } + } + + Trace.TraceInformation("HttpReadCursor.Load: {0} {1}", this, _address.AbsoluteUri); + }, + ex => ex is HttpRequestException || ex is TaskCanceledException, + maxRetries: 5, + initialWaitInterval: TimeSpan.Zero, + waitIncrement: TimeSpan.FromSeconds(10)); + } + } +} \ No newline at end of file diff --git a/src/Catalog/ICatalogGraphPersistence.cs b/src/Catalog/ICatalogGraphPersistence.cs new file mode 100644 index 000000000..119b84c6b --- /dev/null +++ b/src/Catalog/ICatalogGraphPersistence.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; +using System.Threading; +using System.Threading.Tasks; +using VDS.RDF; + +namespace NuGet.Services.Metadata.Catalog +{ + public interface ICatalogGraphPersistence + { + Task SaveGraph(Uri resourceUri, IGraph graph, Uri typeUri, CancellationToken cancellationToken); + Task LoadGraph(Uri resourceUri, CancellationToken cancellationToken); + Uri CreatePageUri(Uri baseAddress, string relativeAddress); + } +} diff --git a/src/Catalog/ICatalogIndexProcessor.cs b/src/Catalog/ICatalogIndexProcessor.cs new file mode 100644 index 000000000..76fa46319 --- /dev/null +++ b/src/Catalog/ICatalogIndexProcessor.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog +{ + /// + /// A processor that runs on entries found on a catalog page. + /// See: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-page + /// + public interface ICatalogIndexProcessor + { + /// + /// Process a single entry from a catalog page. + /// + /// The catalog index entry that should be processed. + /// A task that completes once the entry has been processed. + Task ProcessCatalogIndexEntryAsync(CatalogIndexEntry catalogEntry); + } +} diff --git a/src/Catalog/IHttpRetryStrategy.cs b/src/Catalog/IHttpRetryStrategy.cs new file mode 100644 index 000000000..ea03c3858 --- /dev/null +++ b/src/Catalog/IHttpRetryStrategy.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog +{ + public interface IHttpRetryStrategy + { + Task SendAsync(HttpClient client, Uri address, CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/Catalog/IPackageCatalogItemCreator.cs b/src/Catalog/IPackageCatalogItemCreator.cs new file mode 100644 index 000000000..21b247556 --- /dev/null +++ b/src/Catalog/IPackageCatalogItemCreator.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog +{ + public interface IPackageCatalogItemCreator + { + Task CreateAsync(FeedPackageDetails packageItem, DateTime timestamp, CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/Catalog/IPackagesContainerHandler.cs b/src/Catalog/IPackagesContainerHandler.cs new file mode 100644 index 000000000..f1a689c45 --- /dev/null +++ b/src/Catalog/IPackagesContainerHandler.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog +{ + /// + /// A handler that is run on packages in the packages container. + /// + public interface IPackagesContainerHandler + { + /// + /// Handle a package in the packages container. + /// + /// The package's catalog index entry. + /// The package's blob in the packages container. + /// A task that completes once the package has been handled. + Task ProcessPackageAsync(CatalogIndexEntry packageEntry, ICloudBlockBlob blob); + } +} diff --git a/src/Catalog/Icons/AttemptResult.cs b/src/Catalog/Icons/AttemptResult.cs new file mode 100644 index 000000000..8d56a4e31 --- /dev/null +++ b/src/Catalog/Icons/AttemptResult.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog.Icons +{ + public enum AttemptResult + { + Success, + FailCanRetry, + FailCannotRetry + } +} diff --git a/src/Catalog/Icons/CatalogLeafDataProcessor.cs b/src/Catalog/Icons/CatalogLeafDataProcessor.cs new file mode 100644 index 000000000..54d32500d --- /dev/null +++ b/src/Catalog/Icons/CatalogLeafDataProcessor.cs @@ -0,0 +1,314 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using System.Web; +using Microsoft.Extensions.Logging; +using Microsoft.WindowsAzure.Storage; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog.Icons +{ + public class CatalogLeafDataProcessor : ICatalogLeafDataProcessor + { + private const int MaxExternalIconIngestAttempts = 3; + private const int MaxBlobStorageCopyAttempts = 3; + + private readonly IAzureStorage _packageStorage; + private readonly IIconProcessor _iconProcessor; + private readonly IExternalIconContentProvider _externalIconContentProvider; + private readonly IIconCopyResultCache _iconCopyResultCache; + private readonly ITelemetryService _telemetryService; + private readonly ILogger _logger; + + public CatalogLeafDataProcessor( + IAzureStorage packageStorage, + IIconProcessor iconProcessor, + IExternalIconContentProvider externalIconContentProvider, + IIconCopyResultCache iconCopyResultCache, + ITelemetryService telemetryService, + ILogger logger) + { + _packageStorage = packageStorage ?? throw new ArgumentNullException(nameof(packageStorage)); + _iconProcessor = iconProcessor ?? throw new ArgumentNullException(nameof(iconProcessor)); + _externalIconContentProvider = externalIconContentProvider ?? throw new ArgumentNullException(nameof(externalIconContentProvider)); + _iconCopyResultCache = iconCopyResultCache ?? throw new ArgumentNullException(nameof(iconCopyResultCache)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ProcessPackageDeleteLeafAsync(Storage storage, CatalogCommitItem item, CancellationToken cancellationToken) + { + var targetStoragePath = GetTargetStorageIconPath(item); + await _iconProcessor.DeleteIconAsync(storage, targetStoragePath, cancellationToken, item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()); + // it would be nice to remove the icon copy result from cache for this item, but we don't have an icon URL here, + // so can't remove anything. Will rely on the copy code to catch the copy failure and cleanup the cache appropriately. + } + + public async Task ProcessPackageDetailsLeafAsync(IStorage destinationStorage, IStorage iconCacheStorage, CatalogCommitItem item, string iconUrlString, string iconFile, CancellationToken cancellationToken) + { + var hasExternalIconUrl = !string.IsNullOrWhiteSpace(iconUrlString); + var hasEmbeddedIcon = !string.IsNullOrWhiteSpace(iconFile); + if (hasExternalIconUrl && !hasEmbeddedIcon && Uri.TryCreate(iconUrlString, UriKind.Absolute, out var iconUrl)) + { + using (_logger.BeginScope("Processing icon url {IconUrl}", iconUrl)) + { + await ProcessExternalIconUrlAsync(destinationStorage, iconCacheStorage, item, iconUrl, cancellationToken); + } + } + else if (hasEmbeddedIcon) + { + await ProcessEmbeddedIconAsync(destinationStorage, item, iconFile, cancellationToken); + } + } + + private async Task ProcessExternalIconUrlAsync(IStorage destinationStorage, IStorage iconCacheStorage, CatalogCommitItem item, Uri iconUrl, CancellationToken cancellationToken) + { + _logger.LogInformation("Found external icon url {IconUrl} for {PackageId} {PackageVersion}", + iconUrl, + item.PackageIdentity.Id, + item.PackageIdentity.Version); + if (!IsValidIconUrl(iconUrl)) + { + _logger.LogInformation("Invalid icon URL {IconUrl}", iconUrl); + return; + } + var cachedResult = _iconCopyResultCache.Get(iconUrl); + if (cachedResult != null && await TryTakeFromCache(iconUrl, cachedResult, iconCacheStorage, destinationStorage, item, cancellationToken)) + { + return; + } + using (_telemetryService.TrackExternalIconProcessingDuration(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString())) + { + await CopyIcon(iconUrl, destinationStorage, iconCacheStorage, item, cancellationToken); + } + } + + private async Task CopyIcon(Uri iconUrl, IStorage destinationStorage, IStorage iconCacheStorage, CatalogCommitItem item, CancellationToken cancellationToken) + { + var ingestionResult = await Retry.IncrementalAsync( + async () => await TryIngestExternalIconAsync(item, iconUrl, destinationStorage, cancellationToken), + e => false, + r => r.Result == AttemptResult.FailCanRetry, + MaxExternalIconIngestAttempts, + initialWaitInterval: TimeSpan.FromSeconds(5), + waitIncrement: TimeSpan.FromSeconds(1)); + + if (ingestionResult.Result == AttemptResult.Success) + { + try + { + await _iconCopyResultCache.SaveExternalIcon(iconUrl, ingestionResult.ResultUrl, destinationStorage, iconCacheStorage, cancellationToken); + } + catch (Exception e) + { + // we will report and ignore such exceptions. Failure to store icon will cause the original icon + // to be re-retrieved next time it is encountered. + _logger.LogWarning(0, e, "Failed to store icon in the cache"); + } + } + else + { + var destinationStoragePath = GetTargetStorageIconPath(item); + await _iconProcessor.DeleteIconAsync(destinationStorage, destinationStoragePath, cancellationToken, item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()); + _telemetryService.TrackExternalIconIngestionFailure(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()); + _iconCopyResultCache.SaveExternalCopyFailure(iconUrl); + } + } + + private async Task TryTakeFromCache(Uri iconUrl, ExternalIconCopyResult cachedResult, IStorage iconCacheStorage, IStorage destinationStorage, CatalogCommitItem item, CancellationToken cancellationToken) + { + var targetStoragePath = GetTargetStorageIconPath(item); + if (cachedResult.IsCopySucceeded) + { + _logger.LogInformation("Seen {IconUrl} before, will copy from {CachedLocation}", + iconUrl, + cachedResult.StorageUrl); + var storageUrl = cachedResult.StorageUrl; + var destinationUrl = destinationStorage.ResolveUri(targetStoragePath); + if (storageUrl == destinationUrl) + { + // We came across the package that initially caused the icon to be added to the cache. + // Skipping it. + return true; + } + try + { + await Retry.IncrementalAsync( + async () => await iconCacheStorage.CopyAsync(storageUrl, destinationStorage, destinationUrl, null, cancellationToken), + e => { _logger.LogWarning(0, e, "Exception while copying from cache {StorageUrl}", storageUrl); return true; }, + MaxBlobStorageCopyAttempts, + initialWaitInterval: TimeSpan.FromSeconds(5), + waitIncrement: TimeSpan.FromSeconds(1)); + } + catch (Exception e) + { + _logger.LogWarning(0, e, "Copy from cache failed after {NumRetries} attempts. Falling back to copy from external URL. {StorageUrl}", + MaxBlobStorageCopyAttempts, + storageUrl); + _iconCopyResultCache.Clear(iconUrl); + return false; + } + } + else + { + _logger.LogInformation("Previous copy attempt failed, skipping {IconUrl} for {PackageId} {PackageVersion}", + iconUrl, + item.PackageIdentity.Id, + item.PackageIdentity.Version); + await _iconProcessor.DeleteIconAsync(destinationStorage, targetStoragePath, cancellationToken, item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()); + } + return true; + } + + private async Task ProcessEmbeddedIconAsync(IStorage destinationStorage, CatalogCommitItem item, string iconFile, CancellationToken cancellationToken) + { + var packageFilename = PackageUtility.GetPackageFileName(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()).ToLowerInvariant(); + var packageUri = _packageStorage.ResolveUri(packageFilename); + var packageBlobReference = await _packageStorage.GetCloudBlockBlobReferenceAsync(packageUri); + using (_telemetryService.TrackEmbeddedIconProcessingDuration(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString())) + { + Stream packageStream; + try + { + packageStream = await packageBlobReference.GetStreamAsync(cancellationToken); + } + catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound) + { + _logger.LogWarning("Package blob not found at {PackageUrl}: {Exception}. Will assume package was deleted and skip", + packageUri.AbsoluteUri, + ex); + return; + } + catch (Exception ex) + { + // logging other exceptions here to have proper scope in log message + _logger.LogError("Exception while trying to access package blob {PackageUrl}: {Exception}", + packageUri.AbsoluteUri, + ex); + throw; + } + + using (packageStream) + { + var targetStoragePath = GetTargetStorageIconPath(item); + var resultUrl = await _iconProcessor.CopyEmbeddedIconFromPackageAsync( + packageStream, + iconFile, + destinationStorage, + targetStoragePath, + cancellationToken, + item.PackageIdentity.Id, + item.PackageIdentity.Version.ToNormalizedString()); + } + } + } + + private bool IsValidIconUrl(Uri iconUrl) + { + return iconUrl.Scheme == Uri.UriSchemeHttp || iconUrl.Scheme == Uri.UriSchemeHttps; + } + + private class TryIngestExternalIconAsyncResult + { + public AttemptResult Result { get; private set; } + public Uri ResultUrl { get; private set; } + public static TryIngestExternalIconAsyncResult Fail(AttemptResult failResult) + { + if (failResult == AttemptResult.Success) + { + throw new ArgumentException($"{nameof(failResult)} cannot be {AttemptResult.Success}", nameof(failResult)); + } + + return new TryIngestExternalIconAsyncResult + { + Result = failResult, + ResultUrl = null, + }; + } + public static TryIngestExternalIconAsyncResult FailCannotRetry() => Fail(AttemptResult.FailCannotRetry); + public static TryIngestExternalIconAsyncResult FailCanRetry() => Fail(AttemptResult.FailCanRetry); + public static TryIngestExternalIconAsyncResult Success(Uri resultUrl) + => new TryIngestExternalIconAsyncResult + { + Result = AttemptResult.Success, + ResultUrl = resultUrl ?? throw new ArgumentNullException(nameof(resultUrl)) + }; + } + + private async Task TryIngestExternalIconAsync(CatalogCommitItem item, Uri iconUrl, IStorage destinationStorage, CancellationToken cancellationToken) + { + bool retry; + var resultUrl = (Uri)null; + int maxRetries = 10; + do + { + retry = false; + var getResult = await _externalIconContentProvider.TryGetResponseAsync(iconUrl, cancellationToken); + if (getResult.AttemptResult != AttemptResult.Success) + { + return TryIngestExternalIconAsyncResult.Fail(getResult.AttemptResult); + } + using (var response = getResult.HttpResponseMessage) + { + if (response.StatusCode >= HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.MovedPermanently || response.StatusCode == HttpStatusCode.Found) + { + // normally, HttpClient follows redirects on its own, but there is a limit to it, so if the redirect chain is too long + // it will return 301 or 302, so we'll ignore these specifically. + _logger.LogInformation("Icon url {IconUrl} responded with {ResponseCode}", iconUrl, response.StatusCode); + return response.StatusCode < HttpStatusCode.BadRequest || response.StatusCode == HttpStatusCode.NotFound ? TryIngestExternalIconAsyncResult.FailCannotRetry() : TryIngestExternalIconAsyncResult.FailCanRetry(); + } + if (response.StatusCode == (HttpStatusCode)308) + { + // HttpClient does not seem to support HTTP status code 308, and we have at least one case when we get it: + // http://app.exceptionless.com/images/exceptionless-32.png + // so, we'll had it processed manually + + var newUrl = response.Headers.Location; + + if (iconUrl == newUrl || newUrl == null || !IsValidIconUrl(newUrl)) + { + return TryIngestExternalIconAsyncResult.FailCannotRetry(); + } + + iconUrl = newUrl; + retry = true; + continue; + } + if (!response.IsSuccessStatusCode) + { + _logger.LogWarning("Unexpected response code {ResponseCode} for {IconUrl}", response.StatusCode, iconUrl); + return TryIngestExternalIconAsyncResult.FailCanRetry(); + } + + if (response.Headers.TryGetValues("Content-Type", out var values)) + { + _logger.LogInformation("Reported content type: {ContentType}", values.FirstOrDefault()); + } + using (var iconDataStream = await response.Content.ReadAsStreamAsync()) + { + var targetStoragePath = GetTargetStorageIconPath(item); + resultUrl = await _iconProcessor.CopyIconFromExternalSourceAsync(iconDataStream, destinationStorage, targetStoragePath, cancellationToken, item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()); + } + } + } while (retry && --maxRetries >= 0); + + if (resultUrl == null) + { + return TryIngestExternalIconAsyncResult.FailCannotRetry(); + } + return TryIngestExternalIconAsyncResult.Success(resultUrl); + } + + private static string GetTargetStorageIconPath(CatalogCommitItem item) + { + return $"{item.PackageIdentity.Id.ToLowerInvariant()}/{item.PackageIdentity.Version.ToNormalizedString().ToLowerInvariant()}/icon"; + } + } +} diff --git a/src/Catalog/Icons/ExternalIconContentProvider.cs b/src/Catalog/Icons/ExternalIconContentProvider.cs new file mode 100644 index 000000000..6ecf2f829 --- /dev/null +++ b/src/Catalog/Icons/ExternalIconContentProvider.cs @@ -0,0 +1,85 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Icons +{ + public class ExternalIconContentProvider : IExternalIconContentProvider + { + private readonly IHttpClient _httpResponseMessageProvider; + private readonly ILogger _logger; + + public ExternalIconContentProvider( + IHttpClient httpResponseMessageProvider, + ILogger logger) + { + _httpResponseMessageProvider = httpResponseMessageProvider ?? throw new ArgumentNullException(nameof(httpResponseMessageProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task TryGetResponseAsync(Uri iconUrl, CancellationToken cancellationToken) + { + try + { + return TryGetResponseResult.Success(await _httpResponseMessageProvider.GetAsync(iconUrl, cancellationToken)); + } + catch (HttpRequestException e) when (IsConnectFailure(e)) + { + _logger.LogInformation("Failed to connect to remote host to retrieve the icon"); + } + catch (HttpRequestException e) when (IsDnsFailure(e)) + { + _logger.LogInformation("Failed to resolve DNS name for the icon URL"); + return TryGetResponseResult.FailCannotRetry(); + } + catch (HttpRequestException e) when (IsConnectionClosed(e)) + { + _logger.LogInformation("Connection closed unexpectedly while trying to retrieve the icon"); + } + catch (HttpRequestException e) when (IsTLSSetupFailure(e)) + { + _logger.LogInformation("TLS setup failed while trying to retrieve the icon"); + } + catch (TaskCanceledException e) when (e.CancellationToken != cancellationToken) + { + _logger.LogInformation("Timed out while trying to get the icon data"); + } + catch (HttpRequestException e) + { + _logger.LogError(0, e, "HTTP exception while trying to retrieve icon file"); + } + catch (Exception e) + { + _logger.LogError(0, e, "Exception while trying to retrieve URL: {IconUrl}", iconUrl); + } + return TryGetResponseResult.FailCanRetry(); + } + + private static bool IsConnectFailure(HttpRequestException e) + { + return (e?.InnerException as WebException)?.Status == WebExceptionStatus.ConnectFailure; + } + + private static bool IsDnsFailure(HttpRequestException e) + { + return (e?.InnerException as WebException)?.Status == WebExceptionStatus.NameResolutionFailure; + } + + private static bool IsConnectionClosed(HttpRequestException e) + { + return (e?.InnerException as WebException)?.Status == WebExceptionStatus.ConnectionClosed; + } + + private static bool IsTLSSetupFailure(HttpRequestException e) + { + var innerWebException = e?.InnerException as WebException; + return innerWebException?.Status == WebExceptionStatus.TrustFailure || innerWebException?.Status == WebExceptionStatus.SecureChannelFailure; + } + } +} diff --git a/src/Catalog/Icons/ExternalIconCopyResult.cs b/src/Catalog/Icons/ExternalIconCopyResult.cs new file mode 100644 index 000000000..1f8eb643c --- /dev/null +++ b/src/Catalog/Icons/ExternalIconCopyResult.cs @@ -0,0 +1,59 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Icons +{ + public class ExternalIconCopyResult + { + public static ExternalIconCopyResult Success(Uri sourceUrl, Uri storageUrl) + { + if (sourceUrl == null) + { + throw new ArgumentNullException(nameof(sourceUrl)); + } + + if (storageUrl == null) + { + throw new ArgumentNullException(nameof(storageUrl)); + } + + return new ExternalIconCopyResult + { + SourceUrl = sourceUrl, + StorageUrl = storageUrl, + Expiration = null, // successes don't expire + }; + } + + public static ExternalIconCopyResult Fail(Uri sourceUrl, TimeSpan validityPeriod) + { + if (sourceUrl == null) + { + throw new ArgumentNullException(nameof(sourceUrl)); + } + + if (validityPeriod < TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(validityPeriod), $"{nameof(validityPeriod)} cannot be negative"); + } + + return new ExternalIconCopyResult + { + SourceUrl = sourceUrl, + StorageUrl = null, + Expiration = DateTimeOffset.UtcNow.Add(validityPeriod), + }; + } + + public Uri SourceUrl { get; set; } + public Uri StorageUrl { get; set; } + + /// + /// Expiration time for the fail cache item. + /// + public DateTimeOffset? Expiration { get; set; } + public bool IsCopySucceeded => StorageUrl != null; + } +} diff --git a/src/Catalog/Icons/HttpClientWrapper.cs b/src/Catalog/Icons/HttpClientWrapper.cs new file mode 100644 index 000000000..fe8a9c3a4 --- /dev/null +++ b/src/Catalog/Icons/HttpClientWrapper.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Icons +{ + public class HttpClientWrapper : IHttpClient + { + private readonly HttpClient _httpClient; + + public HttpClientWrapper(HttpClient httpClient) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + } + + public async Task GetAsync(Uri uri, CancellationToken cancellationToken) + { + return await _httpClient.GetAsync(uri, cancellationToken); + } + } +} diff --git a/src/Catalog/Icons/ICatalogLeafDataProcessor.cs b/src/Catalog/Icons/ICatalogLeafDataProcessor.cs new file mode 100644 index 000000000..090d5fc6e --- /dev/null +++ b/src/Catalog/Icons/ICatalogLeafDataProcessor.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog.Icons +{ + public interface ICatalogLeafDataProcessor + { + Task ProcessPackageDeleteLeafAsync(Storage storage, CatalogCommitItem item, CancellationToken cancellationToken); + Task ProcessPackageDetailsLeafAsync(IStorage destinationStorage, IStorage iconCacheStorage, CatalogCommitItem item, string iconUrlString, string iconFile, CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/Catalog/Icons/IExternalIconContentProvider.cs b/src/Catalog/Icons/IExternalIconContentProvider.cs new file mode 100644 index 000000000..d36f4765f --- /dev/null +++ b/src/Catalog/Icons/IExternalIconContentProvider.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Icons +{ + public interface IExternalIconContentProvider + { + Task TryGetResponseAsync(Uri iconUrl, CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/Catalog/Icons/IHttpClient.cs b/src/Catalog/Icons/IHttpClient.cs new file mode 100644 index 000000000..7cf2274da --- /dev/null +++ b/src/Catalog/Icons/IHttpClient.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Icons +{ + public interface IHttpClient + { + Task GetAsync(Uri uri, CancellationToken cancellationToken); + } +} diff --git a/src/Catalog/Icons/IIconCopyResultCache.cs b/src/Catalog/Icons/IIconCopyResultCache.cs new file mode 100644 index 000000000..aec3a22d2 --- /dev/null +++ b/src/Catalog/Icons/IIconCopyResultCache.cs @@ -0,0 +1,48 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog.Icons +{ + /// + /// Interface for external icon copy results. + /// + public interface IIconCopyResultCache + { + /// + /// Checks if there is a known result for a certain external icon URL. + /// + /// External icon URL + /// Previous copy result if we have any, null if nothing is associated with specified . + ExternalIconCopyResult Get(Uri iconUrl); + + /// + /// Copies the successfully retrieved icon blob from destination storage to the cache and + /// takes note of success so it could be reused later. + /// + /// The original URL of an icon. + /// Storage URL where icon was copied to. + /// Storage to which points to. + /// Storage to use for icon cache. + /// Cancellation token. + /// Uri of the icon in the cache storage. + Task SaveExternalIcon(Uri originalIconUrl, Uri storageUrl, IStorage mainDestinationStorage, IStorage cacheStorage, CancellationToken cancellationToken); + + /// + /// Takes note of a failure to copy the external package icon so it's not retried later. + /// + /// The external icon URL. + void SaveExternalCopyFailure(Uri iconUrl); + + + /// + /// Removes the copy result URL (if we have one cached). + /// + /// External icon URL. + void Clear(Uri externalIconUrl); + } +} \ No newline at end of file diff --git a/src/Catalog/Icons/IIconCopyResultCachePersistence.cs b/src/Catalog/Icons/IIconCopyResultCachePersistence.cs new file mode 100644 index 000000000..63986adb1 --- /dev/null +++ b/src/Catalog/Icons/IIconCopyResultCachePersistence.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Icons +{ + public interface IIconCopyResultCachePersistence + { + Task InitializeAsync(CancellationToken cancellationToken); + Task SaveAsync(CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/Catalog/Icons/IIconProcessor.cs b/src/Catalog/Icons/IIconProcessor.cs new file mode 100644 index 000000000..db78abc66 --- /dev/null +++ b/src/Catalog/Icons/IIconProcessor.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog.Icons +{ + public interface IIconProcessor + { + Task CopyEmbeddedIconFromPackageAsync( + Stream packageStream, + string iconFilename, + IStorage destinationStorage, + string destinationStoragePath, + CancellationToken cancellationToken, + string packageId, + string normalizedPackageVersion); + Task CopyIconFromExternalSourceAsync( + Stream iconDataStream, + IStorage destinationStorage, + string destinationStoragePath, + CancellationToken cancellationToken, + string packageId, + string normalizedPackageVersion); + Task DeleteIconAsync( + IStorage destinationStorage, + string destinationStoragePath, + CancellationToken cancellationToken, + string packageId, + string normalizedPackageVersion); + } +} \ No newline at end of file diff --git a/src/Catalog/Icons/IconCopyResultCache.cs b/src/Catalog/Icons/IconCopyResultCache.cs new file mode 100644 index 000000000..68a2b0cf5 --- /dev/null +++ b/src/Catalog/Icons/IconCopyResultCache.cs @@ -0,0 +1,183 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog.Icons +{ + public class IconCopyResultCache : IIconCopyResultCache, IIconCopyResultCachePersistence + { + private const string CacheFilename = "c2i_cache.json"; + + private ConcurrentDictionary _externalIconCopyResults = null; + private ConcurrentDictionary _uriSemaphores = null; + + private readonly IStorage _auxStorage; + private readonly TimeSpan _failCacheTime; + private readonly ILogger _logger; + + public IconCopyResultCache( + IStorage auxStorage, + TimeSpan failCacheTime, + ILogger logger) + { + _auxStorage = auxStorage ?? throw new ArgumentNullException(nameof(auxStorage)); + _failCacheTime = failCacheTime; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + _uriSemaphores = new ConcurrentDictionary(); + } + + public async Task InitializeAsync(CancellationToken cancellationToken) + { + if (_externalIconCopyResults != null) + { + return; + } + + var cacheUrl = _auxStorage.ResolveUri(CacheFilename); + var content = await _auxStorage.LoadAsync(cacheUrl, cancellationToken); + if (content == null) + { + _externalIconCopyResults = new ConcurrentDictionary(); + return; + } + using (var contentStream = content.GetContentStream()) + using (var reader = new StreamReader(contentStream)) + { + var serializer = new JsonSerializer(); + var dictionary = (Dictionary)serializer.Deserialize(reader, typeof(Dictionary)); + _externalIconCopyResults = new ConcurrentDictionary(dictionary); + } + } + + public async Task SaveAsync(CancellationToken cancellationToken) + { + var cacheUrl = _auxStorage.ResolveUri(CacheFilename); + var serialized = JsonConvert.SerializeObject(_externalIconCopyResults); + var content = new StringStorageContent(serialized, contentType: "application/json"); + await _auxStorage.SaveAsync(cacheUrl, content, cancellationToken); + } + + public ExternalIconCopyResult Get(Uri iconUrl) + { + if (_externalIconCopyResults == null) + { + throw new InvalidOperationException("Object was not initialized"); + } + + if (!_externalIconCopyResults.TryGetValue(iconUrl, out var result)) + { + return null; + } + if (!result.IsCopySucceeded && (!result.Expiration.HasValue || result.Expiration.Value < DateTimeOffset.UtcNow)) + { + return null; + } + return result; + } + + public async Task SaveExternalIcon(Uri originalIconUrl, Uri storageUrl, IStorage mainDestinationStorage, IStorage cacheStorage, CancellationToken cancellationToken) + { + if (_externalIconCopyResults == null) + { + throw new InvalidOperationException("Object was not initialized"); + } + + var uriSemaphore = GetUriSemaphore(originalIconUrl); + // Attempting to copy to the same location from multiple sources at the same time will throw, + // so we'll guard the copy attempt with semaphore. + // We'll guard the whole operation so we wouln't even try to copy items to cache more than once. + if (!await uriSemaphore.WaitAsync(TimeSpan.Zero, cancellationToken)) + { + _logger.LogInformation("Failed to enter the semaphore for {IconUrl} immediately, starting to wait", originalIconUrl); + await uriSemaphore.WaitAsync(cancellationToken); + } + try + { + if (_externalIconCopyResults.TryGetValue(originalIconUrl, out var copyResult)) + { + if (copyResult.IsCopySucceeded) + { + return copyResult.StorageUrl; + } + + // if we have failure stored, we'll try to replace it with success, + // now that we've seen one. + } + + var cacheStoragePath = GetCachePath(originalIconUrl); + var cacheUrl = cacheStorage.ResolveUri(cacheStoragePath); + + _logger.LogInformation("Going to store {IconUrl} in cache from {StorageUrl} to {CacheUrl}", + originalIconUrl.AbsoluteUri, + storageUrl.AbsoluteUri, + cacheUrl.AbsoluteUri); + + await mainDestinationStorage.CopyAsync(storageUrl, cacheStorage, cacheUrl, null, cancellationToken); + // Technically, we could get away without storing the success in the dictionary, + // but then each get attempt from the cache would result in HTTP request to cache + // storage that drastically reduces usefulness of the cache (we trade one HTTP request + // for another). + Set(originalIconUrl, ExternalIconCopyResult.Success(originalIconUrl, cacheUrl)); + return cacheUrl; + } + finally + { + uriSemaphore.Release(); + } + } + + private SemaphoreSlim GetUriSemaphore(Uri originalIconUrl) + { + return _uriSemaphores.GetOrAdd(originalIconUrl, _ => new SemaphoreSlim(1, 1)); + } + + public void SaveExternalCopyFailure(Uri iconUrl) + { + if (_externalIconCopyResults == null) + { + throw new InvalidOperationException("Object was not initialized"); + } + + Set(iconUrl, ExternalIconCopyResult.Fail(iconUrl, _failCacheTime)); + } + + private void Set(Uri iconUrl, ExternalIconCopyResult newItem) + { + _externalIconCopyResults.AddOrUpdate(iconUrl, newItem, (_, v) => v.IsCopySucceeded ? v : newItem); // will only overwrite failure results + } + + public void Clear(Uri externalIconUrl) + { + if (_externalIconCopyResults == null) + { + throw new InvalidOperationException("Object was not initialized"); + } + + // Cache results are immutable, so we'll remove looking only at the key + _externalIconCopyResults.TryRemove(externalIconUrl, out var _); + } + + private string GetCachePath(Uri iconUrl) + { + var hash = (byte[])null; + using (var sha512 = new SHA512Managed()) + { + var absoluteUriBytes = Encoding.UTF8.GetBytes(iconUrl.AbsoluteUri); + hash = sha512.ComputeHash(absoluteUriBytes); + } + return "icon-cache/" + BitConverter.ToString(hash).Replace("-", ""); + } + } +} diff --git a/src/Catalog/Icons/IconProcessor.cs b/src/Catalog/Icons/IconProcessor.cs new file mode 100644 index 000000000..49b21fe7f --- /dev/null +++ b/src/Catalog/Icons/IconProcessor.cs @@ -0,0 +1,281 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.WindowsAzure.Storage; +using NuGet.Common; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog.Icons +{ + public class IconProcessor : IIconProcessor + { + private const string DefaultCacheControl = "max-age=120"; + private const int MaxExternalIconSize = 1024 * 1024; // 1 MB + + private readonly ITelemetryService _telemetryService; + private readonly ILogger _logger; + + public IconProcessor( + ITelemetryService telemetryService, + ILogger logger) + { + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task CopyIconFromExternalSourceAsync( + Stream iconDataStream, + IStorage destinationStorage, + string destinationStoragePath, + CancellationToken cancellationToken, + string packageId, + string normalizedPackageVersion) + { + var destinationUri = destinationStorage.ResolveUri(destinationStoragePath); + + var iconData = await GetStreamBytesAsync(iconDataStream, MaxExternalIconSize, cancellationToken); + if (iconData == null) + { + return null; + } + + var contentType = DetermineContentType(iconData, onlyGallerySupported: false); + if (string.IsNullOrWhiteSpace(contentType)) + { + _logger.LogInformation("Failed to determine image type."); + return null; + } + _logger.LogInformation("Content type for {PackageId} {PackageVersion} {ContentType}", packageId, normalizedPackageVersion, contentType); + var content = new ByteArrayStorageContent(iconData, contentType, DefaultCacheControl); + await destinationStorage.SaveAsync(destinationUri, content, cancellationToken); + _telemetryService.TrackExternalIconIngestionSuccess(packageId, normalizedPackageVersion); + return destinationUri; + } + + public async Task DeleteIconAsync( + IStorage destinationStorage, + string destinationStoragePath, + CancellationToken cancellationToken, + string packageId, + string normalizedPackageVersion) + { + _logger.LogInformation("Deleting icon blob {IconPath}", destinationStoragePath); + var iconUri = new Uri(destinationStorage.BaseAddress, destinationStoragePath); + try + { + await destinationStorage.DeleteAsync(iconUri, cancellationToken, new DeleteRequestOptionsWithAccessCondition(AccessCondition.GenerateIfExistsCondition())); + } + catch + { + _telemetryService.TrackIconDeletionFailure(packageId, normalizedPackageVersion); + throw; + } + _telemetryService.TrackIconDeletionSuccess(packageId, normalizedPackageVersion); + } + + public async Task CopyEmbeddedIconFromPackageAsync( + Stream packageStream, + string iconFilename, + IStorage destinationStorage, + string destinationStoragePath, + CancellationToken cancellationToken, + string packageId, + string normalizedPackageVersion) + { + var iconPath = PathUtility.StripLeadingDirectorySeparators(iconFilename); + var destinationUri = destinationStorage.ResolveUri(destinationStoragePath); + + await ExtractAndStoreIconAsync(packageStream, iconPath, destinationStorage, destinationUri, cancellationToken, packageId, normalizedPackageVersion); + return destinationUri; + } + + private async Task ExtractAndStoreIconAsync( + Stream packageStream, + string iconPath, + IStorage destinationStorage, + Uri destinationUri, + CancellationToken cancellationToken, + string packageId, + string normalizedPackageVersion) + { + using (var zipArchive = new ZipArchive(packageStream, ZipArchiveMode.Read, leaveOpen: true)) + { + var iconEntry = zipArchive.Entries.FirstOrDefault(e => e.FullName.Equals(iconPath, StringComparison.InvariantCultureIgnoreCase)); + if (iconEntry != null) + { + using (var iconStream = iconEntry.Open()) + { + _logger.LogInformation("Extracting icon to the destination storage {DestinationUri}", destinationUri); + var iconData = await GetStreamBytesAsync(iconStream, cancellationToken); + // files with embedded icons are expected to only contain image types Gallery allows in + // (jpeg and png). Others are still going to be saved for correctness sake, but we won't + // try to determine their type (they shouldn't have made this far anyway). + var contentType = DetermineContentType(iconData, onlyGallerySupported: true); + _logger.LogInformation("Content type for {PackageId} {PackageVersion} {ContentType}", packageId, normalizedPackageVersion, contentType); + var iconContent = new ByteArrayStorageContent(iconData, contentType, DefaultCacheControl); + await destinationStorage.SaveAsync(destinationUri, iconContent, cancellationToken); + _telemetryService.TrackIconExtractionSuccess(packageId, normalizedPackageVersion); + _logger.LogInformation("Done"); + } + } + else + { + _telemetryService.TrackIconExtractionFailure(packageId, normalizedPackageVersion); + _logger.LogWarning("Zip archive entry {IconPath} does not exist", iconPath); + } + } + } + + private static async Task GetStreamBytesAsync(Stream sourceStream, CancellationToken cancellationToken) + { + using (var ms = new MemoryStream()) + { + await sourceStream.CopyToAsync(ms, 8192, cancellationToken); + return ms.ToArray(); + } + } + + private async Task GetStreamBytesAsync(Stream sourceStream, int maxBytes, CancellationToken cancellationToken) + { + using (var ms = new MemoryStream()) + { + var buffer = new byte[8192]; + var totalBytesRead = 0; + var bytesRead = 0; + do + { + bytesRead = await sourceStream.ReadAsync(buffer, 0, buffer.Length, cancellationToken); + totalBytesRead += bytesRead; + await ms.WriteAsync(buffer, 0, bytesRead, cancellationToken); + } while (bytesRead > 0 && totalBytesRead < maxBytes + 1); + + if (totalBytesRead > maxBytes) + { + _logger.LogInformation("Source data too long, discarding."); + return null; + } + + return ms.ToArray(); + } + } + + /// + /// The PNG file header bytes. All PNG files are expected to have those at the beginning of the file. + /// + /// + /// https://www.w3.org/TR/PNG/#5PNG-file-signature + /// + private static readonly byte[] PngHeader = new byte[] { 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A }; + + /// + /// The JPG file header bytes. + /// + /// + /// Technically, JPEG start with two byte SOI (start of image) segment: FFD8, followed by several other segments or fill bytes. + /// All of the segments start with FF, and fill bytes are FF, so we check the first 3 bytes instead of the first two. + /// https://www.w3.org/Graphics/JPEG/itu-t81.pdf "B.1.1.2 Markers" + /// + private static readonly byte[] JpegHeader = new byte[] { 0xFF, 0xD8, 0xFF }; + + /// + /// The GIF87a file header. + /// + /// + /// https://www.w3.org/Graphics/GIF/spec-gif87.txt + /// + private static readonly byte[] Gif87aHeader = new byte[] { 0x47, 0x49, 0x46, 0x38, 0x37, 0x61 }; + + /// + /// The GIF89a file header. + /// + /// + /// https://www.w3.org/Graphics/GIF/spec-gif89a.txt + /// + private static readonly byte[] Gif89aHeader = new byte[] { 0x47, 0x49, 0x46, 0x38, 0x39, 0x61 }; + + /// + /// The .ico file "header". + /// + /// + /// This is the first 4 bytes of the ICONDIR structure expected for .ico files + /// https://docs.microsoft.com/en-us/previous-versions/ms997538(v=msdn.10) + /// + private static readonly byte[] IcoHeader = new byte[] { 0x00, 0x00, 0x01, 0x00 }; + + private static string DetermineContentType(byte[] imageData, bool onlyGallerySupported) + { + // checks are ordered by format popularity among external icons for existing packages + + if (ArrayStartsWith(imageData, PngHeader)) + { + return "image/png"; + } + + if (ArrayStartsWith(imageData, JpegHeader)) + { + return "image/jpeg"; + } + + if (onlyGallerySupported) + { + return ""; + } + + if (ArrayStartsWith(imageData, IcoHeader)) + { + return "image/x-icon"; + } + + if (ArrayStartsWith(imageData, Gif89aHeader) || ArrayStartsWith(imageData, Gif87aHeader)) + { + return "image/gif"; + } + + if (IsSvgData(imageData)) + { + return "image/svg+xml"; + } + + return ""; + } + + private static bool IsSvgData(byte[] imageData) + { + bool isTextFile = imageData.All(b => b >= 32 || b == '\n' || b == '\f' || b == '\r' || b == '\t'); + if (!isTextFile) + { + return false; + } + + var stringContent = Encoding.UTF8.GetString(imageData); + + return !stringContent.Contains(" _logger; + + public IconsCollector( + Uri index, + ITelemetryService telemetryService, + IStorageFactory targetStorageFactory, + ICatalogClient catalogClient, + ICatalogLeafDataProcessor catalogLeafDataProcessor, + IIconCopyResultCachePersistence iconCopyResultCache, + IStorageFactory iconCacheStorageFactory, + Func httpHandlerFactory, + ILogger logger) + : base(index, telemetryService, httpHandlerFactory, httpClientTimeout: TimeSpan.FromMinutes(5)) + { + _targetStorageFactory = targetStorageFactory ?? throw new ArgumentNullException(nameof(targetStorageFactory)); + _catalogClient = catalogClient ?? throw new ArgumentNullException(nameof(catalogClient)); + _catalogLeafDataProcessor = catalogLeafDataProcessor ?? throw new ArgumentNullException(nameof(catalogLeafDataProcessor)); + _iconCopyResultCache = iconCopyResultCache ?? throw new ArgumentNullException(nameof(iconCopyResultCache)); + _iconCacheStorageFactory = iconCacheStorageFactory ?? throw new ArgumentNullException(nameof(iconCacheStorageFactory)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + protected override Task> CreateBatchesAsync( + IEnumerable catalogItems) + { + var maxCommitTimestamp = catalogItems.Max(x => x.CommitTimeStamp); + + return Task.FromResult>(new[] + { + new CatalogCommitItemBatch( + catalogItems, + key: null, + commitTimestamp: maxCommitTimestamp), + }); + } + + protected override async Task OnProcessBatchAsync( + CollectorHttpClient client, + IEnumerable items, + JToken context, + DateTime commitTimeStamp, + bool isLastBatch, + CancellationToken cancellationToken) + { + await _iconCopyResultCache.InitializeAsync(cancellationToken); + + var filteredItems = items + .GroupBy(i => i.PackageIdentity) // if we have multiple commits for the same package (id AND version) + .Select(g => g.OrderBy(i => i.CommitTimeStamp).ToList()); // group them together for processing in order + var itemsToProcess = new ConcurrentBag>(filteredItems); + var tasks = Enumerable + .Range(1, ServicePointManager.DefaultConnectionLimit) + .Select(_ => ProcessIconsAsync(itemsToProcess, cancellationToken)); + await Task.WhenAll(tasks); + + await _iconCopyResultCache.SaveAsync(cancellationToken); + + return true; + } + + private async Task ProcessIconsAsync( + ConcurrentBag> items, + CancellationToken cancellationToken) + { + await Task.Yield(); + var targetStorage = _targetStorageFactory.Create(); + var iconCacheStorage = _iconCacheStorageFactory.Create(); + + using (_logger.BeginScope("{CallGuid}", Guid.NewGuid())) + while (items.TryTake(out var entries)) + { + var firstItem = entries.First(); + using (_logger.BeginScope("Processing commits for {PackageId} {PackageVersion}", firstItem.PackageIdentity.Id, firstItem.PackageIdentity.Version)) + { + foreach (var item in entries) + { + if (item.IsPackageDetails) + { + PackageDetailsCatalogLeaf leaf; + try + { + leaf = await _catalogClient.GetPackageDetailsLeafAsync(item.Uri.AbsoluteUri); + } + catch (Exception e) + { + _logger.LogError(0, e, "Error while trying to retrieve catalog leaf {LeafUrl}", item.Uri.AbsoluteUri); + throw; + } + await _catalogLeafDataProcessor.ProcessPackageDetailsLeafAsync(targetStorage, iconCacheStorage, item, leaf.IconUrl, leaf.IconFile, cancellationToken); + } + else if (item.IsPackageDelete) + { + await _catalogLeafDataProcessor.ProcessPackageDeleteLeafAsync(targetStorage, item, cancellationToken); + } + } + } + } + } + } +} diff --git a/src/Catalog/Icons/TryGetResponseResult.cs b/src/Catalog/Icons/TryGetResponseResult.cs new file mode 100644 index 000000000..391507f13 --- /dev/null +++ b/src/Catalog/Icons/TryGetResponseResult.cs @@ -0,0 +1,40 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Net.Http; + +namespace NuGet.Services.Metadata.Catalog.Icons +{ + public class TryGetResponseResult + { + public HttpResponseMessage HttpResponseMessage { get; set; } + public AttemptResult AttemptResult; + + public static TryGetResponseResult Success(HttpResponseMessage httpResponseMessage) + { + return new TryGetResponseResult + { + AttemptResult = AttemptResult.Success, + HttpResponseMessage = httpResponseMessage, + }; + } + + public static TryGetResponseResult FailCanRetry() + { + return new TryGetResponseResult + { + AttemptResult = AttemptResult.FailCanRetry, + HttpResponseMessage = null, + }; + } + + public static TryGetResponseResult FailCannotRetry() + { + return new TryGetResponseResult + { + AttemptResult = AttemptResult.FailCannotRetry, + HttpResponseMessage = null, + }; + } + } +} diff --git a/src/Catalog/JsonLdIntegration/JsonLdReader.cs b/src/Catalog/JsonLdIntegration/JsonLdReader.cs new file mode 100644 index 000000000..3c8f97824 --- /dev/null +++ b/src/Catalog/JsonLdIntegration/JsonLdReader.cs @@ -0,0 +1,204 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System; +using System.IO; +using VDS.RDF; +using VDS.RDF.Parsing.Handlers; + +namespace NuGet.Services.Metadata.Catalog.JsonLDIntegration +{ + public class JsonLdReader : IRdfReader + { + public JsonLdReader() + { + if (Warning == null) + { + // this event looks a little brain damaged + } + } + + public void Load(IRdfHandler handler, string filename) + { + Load(handler, new StreamReader(filename)); + } + + public void Load(IRdfHandler handler, TextReader input) + { + bool finished = false; + try + { + // Tell handler we starting parsing + handler.StartRdf(); + + // Perform actual parsing + using (JsonReader jsonReader = new JsonTextReader(input)) + { + jsonReader.DateParseHandling = DateParseHandling.None; + + JToken json = JToken.Load(jsonReader); + + foreach (JObject subjectJObject in json) + { + string subject = subjectJObject["@id"].ToString(); + + JToken type; + if (subjectJObject.TryGetValue("@type", out type)) + { + if (type is JArray) + { + foreach (JToken t in (JArray) type) + { + if (!HandleTriple(handler, subject, "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", t.ToString(), null, false)) return; + } + } + else + { + if (!HandleTriple(handler, subject, "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", type.ToString(), null, false)) return; + } + } + + foreach (JProperty property in subjectJObject.Properties()) + { + if (property.Name == "@id" || property.Name == "@type") + { + continue; + } + + foreach (JObject objectJObject in property.Value) + { + JToken id; + JToken value; + if (objectJObject.TryGetValue("@id", out id)) + { + if (!HandleTriple(handler, subject, property.Name, id.ToString(), null, false)) return; + } + else if (objectJObject.TryGetValue("@value", out value)) + { + string datatype = null; + JToken datatypeJToken; + if (objectJObject.TryGetValue("@type", out datatypeJToken)) + { + datatype = datatypeJToken.ToString(); + } + else + { + switch (value.Type) + { + case JTokenType.Boolean: + datatype = "http://www.w3.org/2001/XMLSchema#boolean"; + break; + case JTokenType.Float: + datatype = "http://www.w3.org/2001/XMLSchema#double"; + break; + case JTokenType.Integer: + datatype = "http://www.w3.org/2001/XMLSchema#integer"; + break; + } + } + + if (!HandleTriple(handler, subject, property.Name, value.ToString(), datatype, true)) return; + } + } + } + } + } + + // Tell handler we've finished parsing + finished = true; + handler.EndRdf(true); + } + catch + { + // Catch all block to fulfill the IRdfHandler contract of informing the handler when the parsing has ended with failure + finished = true; + handler.EndRdf(false); + throw; + } + finally + { + // Finally block handles the case where we exit the parsing loop early because the handler indicated it did not want + // to receive further triples. In this case finished will be set to false and we need to inform the handler we're are done + if (!finished) + { + handler.EndRdf(true); + } + } + } + + public void Load(IRdfHandler handler, StreamReader input) + { + Load(handler, (TextReader)input); + } + + public void Load(IGraph g, string filename) + { + Load(g, new StreamReader(filename)); + } + + public void Load(IGraph g, TextReader input) + { + Load(new GraphHandler(g), input); + } + + public void Load(IGraph g, StreamReader input) + { + Load(g, (TextReader)input); + } + + public event RdfReaderWarning Warning; + + /// + /// Creates and handles a triple + /// + /// Handler + /// Subject + /// Predicate + /// Object + /// Object Datatype + /// isLiteral Object + /// True if parsing should continue, false otherwise + bool HandleTriple(IRdfHandler handler, string subject, string predicate, string obj, string datatype, bool isLiteral) + { + INode subjectNode; + if (subject.StartsWith("_")) + { + string nodeId = subject.Substring(subject.IndexOf(":") + 1); + subjectNode = handler.CreateBlankNode(nodeId); + } + else + { + subjectNode = handler.CreateUriNode(new Uri(subject)); + } + + INode predicateNode = handler.CreateUriNode(new Uri(predicate)); + + INode objNode; + if (isLiteral) + { + if (datatype == "http://www.w3.org/2001/XMLSchema#boolean") + { + // sometimes newtonsoft.json appears to return boolean as string True and dotNetRdf doesn't appear to recognize that + obj = ((string)obj).ToLowerInvariant(); + } + + objNode = (datatype == null) ? handler.CreateLiteralNode((string)obj) : handler.CreateLiteralNode((string)obj, new Uri(datatype)); + } + else + { + if (obj.StartsWith("_")) + { + string nodeId = obj.Substring(obj.IndexOf(":") + 1); + objNode = handler.CreateBlankNode(nodeId); + } + else + { + objNode = handler.CreateUriNode(new Uri(obj)); + } + } + + return handler.HandleTriple(new Triple(subjectNode, predicateNode, objNode)); + } + } +} diff --git a/src/Catalog/JsonLdIntegration/JsonLdWriter.cs b/src/Catalog/JsonLdIntegration/JsonLdWriter.cs new file mode 100644 index 000000000..34e4ec734 --- /dev/null +++ b/src/Catalog/JsonLdIntegration/JsonLdWriter.cs @@ -0,0 +1,223 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using VDS.RDF; +using Newtonsoft.Json; + +namespace NuGet.Services.Metadata.Catalog.JsonLDIntegration +{ + public class JsonLdWriter : IRdfWriter + { + public JsonLdWriter() + { + if (Warning == null) + { + // this event looks a little brain damaged + } + } + + public void Save(IGraph g, TextWriter output) + { + JToken flattened = MakeExpandedForm(g); + + output.Write(flattened.ToString(Formatting.None, new JsonConverter[0])); + output.Flush(); + } + + public void Save(IGraph g, string filename) + { + Save(g, new StreamWriter(filename)); + } + + public event RdfWriterWarning Warning; + + const string First = "http://www.w3.org/1999/02/22-rdf-syntax-ns#first"; + const string Rest = "http://www.w3.org/1999/02/22-rdf-syntax-ns#rest"; + const string Nil = "http://www.w3.org/1999/02/22-rdf-syntax-ns#nil"; + + static IDictionary> GetLists(IGraph graph) + { + INode first = graph.CreateUriNode(new Uri(First)); + INode rest = graph.CreateUriNode(new Uri(Rest)); + INode nil = graph.CreateUriNode(new Uri(Nil)); + + IEnumerable ends = graph.GetTriplesWithPredicateObject(rest, nil); + + IDictionary> lists = new Dictionary>(); + + foreach (Triple end in ends) + { + List list = new List(); + + Triple iterator = graph.GetTriplesWithSubjectPredicate(end.Subject, first).First(); + + INode head = iterator.Subject; + + while (true) + { + list.Add(iterator.Object); + + IEnumerable restTriples = graph.GetTriplesWithPredicateObject(rest, iterator.Subject); + + if (restTriples.Count() == 0) + { + break; + } + + iterator = graph.GetTriplesWithSubjectPredicate(restTriples.First().Subject, first).First(); + + head = iterator.Subject; + } + + list.Reverse(); + + lists.Add(head, list); + } + + return lists; + } + + static bool IsListNode(INode subject, IGraph graph) + { + INode rest = graph.CreateUriNode(new Uri(Rest)); + return (graph.GetTriplesWithSubjectPredicate(subject, rest).Count() > 0); + } + + JToken MakeExpandedForm(IGraph graph) + { + IDictionary> lists = GetLists(graph); + + IDictionary subjects = new Dictionary(); + + foreach (Triple triple in graph.Triples.Where((t) => !IsListNode(t.Subject, graph))) + { + string subject = triple.Subject.ToString(); + string predicate = triple.Predicate.ToString(); + + if (predicate == "http://www.w3.org/1999/02/22-rdf-syntax-ns#type") + { + predicate = "@type"; + } + + JObject properties; + if (!subjects.TryGetValue(subject, out properties)) + { + properties = new JObject(); + properties.Add("@id", subject); + subjects.Add(subject, properties); + } + + JArray objects; + JToken o; + if (!properties.TryGetValue(predicate, out o)) + { + objects = new JArray(); + properties.Add(predicate, objects); + } + else + { + objects = (JArray)o; + } + + if (predicate == "@type") + { + objects.Add(triple.Object.ToString()); + } + else + { + if (lists.ContainsKey(triple.Object)) + { + objects.Add(MakeList(lists[triple.Object])); + } + else + { + objects.Add(MakeObject(triple.Object)); + } + } + } + + JArray result = new JArray(); + foreach (JObject subject in subjects.Values) + { + result.Add(subject); + } + + return result; + } + + static JToken MakeList(List nodes) + { + JArray list = new JArray(); + + foreach (INode node in nodes) + { + list.Add(MakeObject(node)); + } + + return new JObject { { "@list", list } }; + } + + static JToken MakeObject(INode node) + { + if (node is IUriNode) + { + return new JObject { { "@id", node.ToString() } }; + } + else if (node is IBlankNode) + { + return new JObject { { "@id", node.ToString() } }; + } + else + { + return MakeLiteralObject((ILiteralNode)node); + } + } + + static JObject MakeLiteralObject(ILiteralNode node) + { + if (node.DataType == null) + { + return new JObject { { "@value", node.Value } }; + } + else + { + string dataType = node.DataType.ToString(); + + switch (dataType) + { + case "http://www.w3.org/2001/XMLSchema#integer": + return new JObject { { "@value", int.Parse(node.Value) } }; + + case "http://www.w3.org/2001/XMLSchema#boolean": + return new JObject { { "@value", bool.Parse(node.Value) } }; + + case "http://www.w3.org/2001/XMLSchema#decimal": + return new JObject { { "@value", decimal.Parse(node.Value) } }; + + case "http://www.w3.org/2001/XMLSchema#long": + return new JObject { { "@value", long.Parse(node.Value) } }; + + case "http://www.w3.org/2001/XMLSchema#short": + return new JObject { { "@value", short.Parse(node.Value) } }; + + case "http://www.w3.org/2001/XMLSchema#float": + return new JObject { { "@value", float.Parse(node.Value) } }; + + case "http://www.w3.org/2001/XMLSchema#double": + return new JObject { { "@value", double.Parse(node.Value) } }; + + default: + return new JObject + { + { "@value", node.Value }, + { "@type", dataType } + }; + } + } + } + } +} diff --git a/src/Catalog/MemoryCursor.cs b/src/Catalog/MemoryCursor.cs new file mode 100644 index 000000000..2595deffc --- /dev/null +++ b/src/Catalog/MemoryCursor.cs @@ -0,0 +1,38 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog +{ + public class MemoryCursor : ReadWriteCursor + { + public static DateTime MinValue = DateTime.MinValue.ToUniversalTime(); + public static DateTime MaxValue = DateTime.MaxValue.ToUniversalTime(); + + public static MemoryCursor CreateMin() { return new MemoryCursor(MinValue); } + public static MemoryCursor CreateMax() { return new MemoryCursor(MaxValue); } + + public MemoryCursor(DateTime value) + { + Value = value; + } + + public MemoryCursor() + { + // TODO: Complete member initialization + } + + public override Task LoadAsync(CancellationToken cancellationToken) + { + return Task.FromResult(0); + } + + public override Task SaveAsync(CancellationToken cancellationToken) + { + return Task.FromResult(0); + } + } +} \ No newline at end of file diff --git a/src/Catalog/NuGet.Services.Metadata.Catalog.csproj b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj new file mode 100644 index 000000000..26ab82269 --- /dev/null +++ b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj @@ -0,0 +1,298 @@ + + + + + + Debug + AnyCPU + {E97F23B8-ECB0-4AFA-B00C-015C39395FEF} + Library + Properties + NuGet.Services.Metadata.Catalog + NuGet.Services.Metadata.Catalog + v4.7.2 + 512 + + true + win + + + .NET Foundation + https://github.com/NuGet/NuGet.Services.Metadata/blob/master/LICENSE + https://github.com/NuGet/NuGet.Services.Metadata + Create, edit, or read the package metadata catalog. + nuget;services;search;catalog;metadata;collector + Copyright .NET Foundation + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + true + bin\x64\Debug\ + TRACE;DEBUG + full + x64 + prompt + MinimumRecommendedRules.ruleset + + + bin\x64\Release\ + TRACE + true + pdbonly + x64 + prompt + MinimumRecommendedRules.ruleset + + + + + + + + + + + + + ..\..\packages\System.Runtime.InteropServices.RuntimeInformation.4.0.0\lib\net45\System.Runtime.InteropServices.RuntimeInformation.dll + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + True + True + Strings.resx + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ResXFileCodeGenerator + Strings.Designer.cs + + + + Designer + + + + + + + + + + + + + + 0.3.0 + runtime; build; native; contentfiles; analyzers + all + + + 1.0.8.3533 + + + 4.4.5-dev-3612899 + + + 2.75.0 + + + 1.0.6 + + + 0.7.1 + + + 3.1.0 + + + 4.8.0 + runtime; build; native; contentfiles; analyzers + all + + + 2.75.0 + + + 9.3.3 + + + + + {d44c2e89-2d98-44bd-8712-8ccbe4e67c9c} + NuGet.Protocol.Catalog + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + + \ No newline at end of file diff --git a/src/Catalog/NuGet.Services.Metadata.Catalog.csproj.DotSettings b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj.DotSettings new file mode 100644 index 000000000..73e96563f --- /dev/null +++ b/src/Catalog/NuGet.Services.Metadata.Catalog.csproj.DotSettings @@ -0,0 +1,2 @@ + + CSharp60 \ No newline at end of file diff --git a/src/Catalog/NupkgMetadata.cs b/src/Catalog/NupkgMetadata.cs new file mode 100644 index 000000000..92c175660 --- /dev/null +++ b/src/Catalog/NupkgMetadata.cs @@ -0,0 +1,24 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Xml.Linq; + +namespace NuGet.Services.Metadata.Catalog +{ + public class NupkgMetadata + { + public XDocument Nuspec { get; } + public IEnumerable Entries { get; } + public long PackageSize { get; } + public string PackageHash { get; } + + public NupkgMetadata(XDocument nuspec, IEnumerable entries, long packageSize, string packageHash) + { + Nuspec = nuspec; + Entries = entries; + PackageSize = packageSize; + PackageHash = packageHash; + } + } +} \ No newline at end of file diff --git a/src/Catalog/PackageCatalog.cs b/src/Catalog/PackageCatalog.cs new file mode 100644 index 000000000..bdd86107f --- /dev/null +++ b/src/Catalog/PackageCatalog.cs @@ -0,0 +1,80 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Globalization; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using VDS.RDF; + +namespace NuGet.Services.Metadata.Catalog +{ + public static class PackageCatalog + { + public static IGraph CreateCommitMetadata(Uri indexUri, CommitMetadata commitMetadata) + { + IGraph graph = new Graph(); + + if (commitMetadata.LastCreated != null) + { + graph.Assert( + graph.CreateUriNode(indexUri), + graph.CreateUriNode(Schema.Predicates.LastCreated), + graph.CreateLiteralNode(commitMetadata.LastCreated.Value.ToString("O"), Schema.DataTypes.DateTime)); + } + if (commitMetadata.LastEdited != null) + { + graph.Assert( + graph.CreateUriNode(indexUri), + graph.CreateUriNode(Schema.Predicates.LastEdited), + graph.CreateLiteralNode(commitMetadata.LastEdited.Value.ToString("O"), Schema.DataTypes.DateTime)); + } + if (commitMetadata.LastDeleted != null) + { + graph.Assert( + graph.CreateUriNode(indexUri), + graph.CreateUriNode(Schema.Predicates.LastDeleted), + graph.CreateLiteralNode(commitMetadata.LastDeleted.Value.ToString("O"), Schema.DataTypes.DateTime)); + } + + return graph; + } + + public static async Task ReadCommitMetadata(CatalogWriterBase writer, CancellationToken cancellationToken) + { + CommitMetadata commitMetadata = new CommitMetadata(); + + string json = await writer.Storage.LoadStringAsync(writer.RootUri, cancellationToken); + + if (json != null) + { + JObject obj; + + using (JsonReader jsonReader = new JsonTextReader(new StringReader(json))) + { + jsonReader.DateParseHandling = DateParseHandling.None; + obj = JObject.Load(jsonReader); + } + + commitMetadata.LastCreated = TryGetDateTimeFromJObject(obj, "nuget:lastCreated"); + commitMetadata.LastEdited = TryGetDateTimeFromJObject(obj, "nuget:lastEdited"); + commitMetadata.LastDeleted = TryGetDateTimeFromJObject(obj, "nuget:lastDeleted"); + } + + return commitMetadata; + } + + private static DateTime? TryGetDateTimeFromJObject(JObject target, string propertyName) + { + JToken token; + if (target.TryGetValue(propertyName, out token)) + { + return DateTime.Parse(token.ToString(), null, DateTimeStyles.RoundtripKind); + } + return null; + } + } +} \ No newline at end of file diff --git a/src/Catalog/PackageCatalogItem.cs b/src/Catalog/PackageCatalogItem.cs new file mode 100644 index 000000000..b7170b7f6 --- /dev/null +++ b/src/Catalog/PackageCatalogItem.cs @@ -0,0 +1,276 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Newtonsoft.Json.Linq; +using NuGet.Services.Entities; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; +using VDS.RDF; + +namespace NuGet.Services.Metadata.Catalog +{ + public class PackageCatalogItem : AppendOnlyCatalogItem + { + private string _id; + private string _fullVersion; + private string _normalizedVersion; + + // These properties are public only to facilitate testing. + public NupkgMetadata NupkgMetadata { get; } + public DateTime? CreatedDate { get; } + public DateTime? LastEditedDate { get; } + public DateTime? PublishedDate { get; } + public PackageDeprecationItem Deprecation { get; } + public IList Vulnerabilities { get; } + + public PackageCatalogItem( + NupkgMetadata nupkgMetadata, + DateTime? createdDate = null, + DateTime? lastEditedDate = null, + DateTime? publishedDate = null, + string licenseNames = null, + string licenseReportUrl = null, + PackageDeprecationItem deprecation = null, + IList vulnerabilities = null) + { + NupkgMetadata = nupkgMetadata; + CreatedDate = createdDate; + LastEditedDate = lastEditedDate; + PublishedDate = publishedDate; + Deprecation = deprecation; + Vulnerabilities = vulnerabilities; + } + + public override IGraph CreateContentGraph(CatalogContext context) + { + IGraph graph = Utils.CreateNuspecGraph(NupkgMetadata.Nuspec, GetBaseAddress().ToString(), normalizeXml: true); + + // catalog infrastructure fields + INode rdfTypePredicate = graph.CreateUriNode(Schema.Predicates.Type); + INode permanentType = graph.CreateUriNode(Schema.DataTypes.Permalink); + Triple resource = graph.GetTriplesWithPredicateObject(rdfTypePredicate, graph.CreateUriNode(GetItemType())).First(); + graph.Assert(resource.Subject, rdfTypePredicate, permanentType); + + // published + INode publishedPredicate = graph.CreateUriNode(Schema.Predicates.Published); + DateTime published = PublishedDate ?? TimeStamp; + graph.Assert(resource.Subject, publishedPredicate, graph.CreateLiteralNode(published.ToString("O"), Schema.DataTypes.DateTime)); + + // listed + INode listedPredicate = graph.CreateUriNode(Schema.Predicates.Listed); + Boolean listed = GetListed(published); + graph.Assert(resource.Subject, listedPredicate, graph.CreateLiteralNode(listed.ToString(), Schema.DataTypes.Boolean)); + + // created + INode createdPredicate = graph.CreateUriNode(Schema.Predicates.Created); + DateTime created = CreatedDate ?? TimeStamp; + graph.Assert(resource.Subject, createdPredicate, graph.CreateLiteralNode(created.ToString("O"), Schema.DataTypes.DateTime)); + + // lastEdited + INode lastEditedPredicate = graph.CreateUriNode(Schema.Predicates.LastEdited); + DateTime lastEdited = LastEditedDate ?? DateTime.MinValue; + graph.Assert(resource.Subject, lastEditedPredicate, graph.CreateLiteralNode(lastEdited.ToString("O"), Schema.DataTypes.DateTime)); + + // entries + if (NupkgMetadata.Entries != null) + { + INode packageEntryPredicate = graph.CreateUriNode(Schema.Predicates.PackageEntry); + INode packageEntryType = graph.CreateUriNode(Schema.DataTypes.PackageEntry); + INode fullNamePredicate = graph.CreateUriNode(Schema.Predicates.FullName); + INode namePredicate = graph.CreateUriNode(Schema.Predicates.Name); + INode lengthPredicate = graph.CreateUriNode(Schema.Predicates.Length); + INode compressedLengthPredicate = graph.CreateUriNode(Schema.Predicates.CompressedLength); + + foreach (PackageEntry entry in NupkgMetadata.Entries) + { + Uri entryUri = new Uri(resource.Subject.ToString() + "#" + entry.FullName); + + INode entryNode = graph.CreateUriNode(entryUri); + + graph.Assert(resource.Subject, packageEntryPredicate, entryNode); + graph.Assert(entryNode, rdfTypePredicate, packageEntryType); + graph.Assert(entryNode, fullNamePredicate, graph.CreateLiteralNode(entry.FullName)); + graph.Assert(entryNode, namePredicate, graph.CreateLiteralNode(entry.Name)); + graph.Assert(entryNode, lengthPredicate, graph.CreateLiteralNode(entry.Length.ToString(), Schema.DataTypes.Integer)); + graph.Assert(entryNode, compressedLengthPredicate, graph.CreateLiteralNode(entry.CompressedLength.ToString(), Schema.DataTypes.Integer)); + } + } + + // packageSize and packageHash + graph.Assert(resource.Subject, graph.CreateUriNode(Schema.Predicates.PackageSize), graph.CreateLiteralNode(NupkgMetadata.PackageSize.ToString(), Schema.DataTypes.Integer)); + graph.Assert(resource.Subject, graph.CreateUriNode(Schema.Predicates.PackageHash), graph.CreateLiteralNode(NupkgMetadata.PackageHash)); + graph.Assert(resource.Subject, graph.CreateUriNode(Schema.Predicates.PackageHashAlgorithm), graph.CreateLiteralNode(Constants.Sha512)); + + // identity and version + SetIdVersionFromGraph(graph); + + // deprecation + if (Deprecation != null) + { + // assert deprecation root node to subject + var deprecationPredicate = graph.CreateUriNode(Schema.Predicates.Deprecation); + var deprecationRootNode = graph.CreateUriNode(new Uri(resource.Subject.ToString() + "#deprecation")); + graph.Assert(resource.Subject, deprecationPredicate, deprecationRootNode); + + // assert reasons to deprecation root node + var deprecationReasonRootNode = graph.CreateUriNode(Schema.Predicates.Reasons); + foreach (var reason in Deprecation.Reasons) + { + var reasonNode = graph.CreateLiteralNode(reason); + graph.Assert(deprecationRootNode, deprecationReasonRootNode, reasonNode); + } + + // assert message to deprecation root node + if (Deprecation.Message != null) + { + graph.Assert( + deprecationRootNode, + graph.CreateUriNode(Schema.Predicates.Message), + graph.CreateLiteralNode(Deprecation.Message)); + } + + if (Deprecation.AlternatePackageId != null) + { + // assert alternate package root node to deprecation root node + var deprecationAlternatePackagePredicate = graph.CreateUriNode(Schema.Predicates.AlternatePackage); + var deprecationAlternatePackageRootNode = graph.CreateUriNode(new Uri(resource.Subject.ToString() + "#deprecation/alternatePackage")); + graph.Assert(deprecationRootNode, deprecationAlternatePackagePredicate, deprecationAlternatePackageRootNode); + + // assert id to alternate package root node + graph.Assert( + deprecationAlternatePackageRootNode, + graph.CreateUriNode(Schema.Predicates.Id), + graph.CreateLiteralNode(Deprecation.AlternatePackageId)); + + // assert version range to alternate package root node + graph.Assert( + deprecationAlternatePackageRootNode, + graph.CreateUriNode(Schema.Predicates.Range), + graph.CreateLiteralNode(Deprecation.AlternatePackageRange)); + } + } + + // vulnerabilities + if (Vulnerabilities != null) + { + INode vulnerabilityPredicate = graph.CreateUriNode(Schema.Predicates.Vulnerability); + INode vulnerabilityType = graph.CreateUriNode(Schema.DataTypes.Vulnerability); + INode advisoryUrlPredicate = graph.CreateUriNode(Schema.Predicates.AdvisoryUrl); + INode severityPredicate = graph.CreateUriNode(Schema.Predicates.Severity); + + foreach (PackageVulnerabilityItem vulnerability in Vulnerabilities) + { + Uri vulnerabilityUri = new Uri(resource.Subject.ToString() + "#vulnerability/GitHub/" + vulnerability.GitHubDatabaseKey); + + INode vulnerabilityNode = graph.CreateUriNode(vulnerabilityUri); + + graph.Assert(resource.Subject, vulnerabilityPredicate, vulnerabilityNode); + graph.Assert(vulnerabilityNode, rdfTypePredicate, vulnerabilityType); + graph.Assert(vulnerabilityNode, advisoryUrlPredicate, graph.CreateLiteralNode(vulnerability.AdvisoryUrl)); + graph.Assert(vulnerabilityNode, severityPredicate, graph.CreateLiteralNode(vulnerability.Severity)); + } + } + + return graph; + } + + public static bool GetListed(DateTime published) + { + // if the published date is 1900/01/01, then the package is unlisted + if (published.ToUniversalTime() == Constants.UnpublishedDate) + { + return false; + } + + return true; + } + + protected void SetIdVersionFromGraph(IGraph graph) + { + INode idPredicate = graph.CreateUriNode(Schema.Predicates.Id); + INode versionPredicate = graph.CreateUriNode(Schema.Predicates.Version); + + INode rdfTypePredicate = graph.CreateUriNode(Schema.Predicates.Type); + Triple resource = graph.GetTriplesWithPredicateObject(rdfTypePredicate, graph.CreateUriNode(GetItemType())).First(); + Triple id = graph.GetTriplesWithSubjectPredicate(resource.Subject, idPredicate).FirstOrDefault(); + if (id != null) + { + _id = ((ILiteralNode)id.Object).Value; + } + + Triple version = graph.GetTriplesWithSubjectPredicate(resource.Subject, versionPredicate).FirstOrDefault(); + if (version != null) + { + _fullVersion = ((ILiteralNode)version.Object).Value; + _normalizedVersion = NuGetVersionUtility.NormalizeVersion(_fullVersion); + } + } + + public override StorageContent CreateContent(CatalogContext context) + { + // metadata from nuspec + + using (IGraph graph = CreateContentGraph(context)) + { + // catalog infrastructure fields + INode rdfTypePredicate = graph.CreateUriNode(Schema.Predicates.Type); + INode timeStampPredicate = graph.CreateUriNode(Schema.Predicates.CatalogTimeStamp); + INode commitIdPredicate = graph.CreateUriNode(Schema.Predicates.CatalogCommitId); + + Triple resource = graph.GetTriplesWithPredicateObject(rdfTypePredicate, graph.CreateUriNode(GetItemType())).First(); + graph.Assert(resource.Subject, timeStampPredicate, graph.CreateLiteralNode(TimeStamp.ToString("O"), Schema.DataTypes.DateTime)); + graph.Assert(resource.Subject, commitIdPredicate, graph.CreateLiteralNode(CommitId.ToString())); + + if (graph.GetTriples(Schema.Predicates.Deprecation).Count() > 1) + { + throw new ArgumentException("Package catalog items can only have a single deprecation."); + } + + // create JSON content + JObject frame = context.GetJsonLdContext("context.PackageDetails.json", GetItemType()); + + StorageContent content = new StringStorageContent(Utils.CreateArrangedJson(graph, frame), "application/json", "no-store"); + + return content; + } + } + + + public override Uri GetItemType() + { + return Schema.DataTypes.PackageDetails; + } + + public override IGraph CreatePageContent(CatalogContext context) + { + Uri resourceUri = new Uri(GetBaseAddress() + GetRelativeAddress()); + + Graph graph = new Graph(); + + INode subject = graph.CreateUriNode(resourceUri); + + INode idPredicate = graph.CreateUriNode(Schema.Predicates.Id); + INode versionPredicate = graph.CreateUriNode(Schema.Predicates.Version); + + if (_id != null) + { + graph.Assert(subject, idPredicate, graph.CreateLiteralNode(_id)); + } + + if (_fullVersion != null) + { + graph.Assert(subject, versionPredicate, graph.CreateLiteralNode(_fullVersion)); + } + + return graph; + } + + protected override string GetItemIdentity() + { + return (_id + "." + _normalizedVersion).ToLowerInvariant(); + } + } +} \ No newline at end of file diff --git a/src/Catalog/PackageCatalogItemCreator.cs b/src/Catalog/PackageCatalogItemCreator.cs new file mode 100644 index 000000000..fdcd09f54 --- /dev/null +++ b/src/Catalog/PackageCatalogItemCreator.cs @@ -0,0 +1,268 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog +{ + public sealed class PackageCatalogItemCreator : IPackageCatalogItemCreator + { + private readonly HttpClient _httpClient; + private readonly ILogger _logger; + private readonly IAzureStorage _storage; + private readonly ITelemetryService _telemetryService; + + private PackageCatalogItemCreator( + HttpClient httpClient, + ITelemetryService telemetryService, + ILogger logger, + IStorage storage) + { + _httpClient = httpClient; + _telemetryService = telemetryService; + _logger = logger; + _storage = storage as IAzureStorage; + } + + public static PackageCatalogItemCreator Create( + HttpClient httpClient, + ITelemetryService telemetryService, + ILogger logger, + IStorage storage) + { + if (httpClient == null) + { + throw new ArgumentNullException(nameof(httpClient)); + } + + if (telemetryService == null) + { + throw new ArgumentNullException(nameof(telemetryService)); + } + + if (logger == null) + { + throw new ArgumentNullException(nameof(logger)); + } + + return new PackageCatalogItemCreator(httpClient, telemetryService, logger, storage); + } + + public async Task CreateAsync( + FeedPackageDetails packageItem, + DateTime timestamp, + CancellationToken cancellationToken) + { + if (packageItem == null) + { + throw new ArgumentNullException(nameof(packageItem)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + PackageCatalogItem item = null; + + _logger.LogInformation( + "Creating package catalog item for {Id} {Version}", + packageItem.PackageId, + packageItem.PackageNormalizedVersion); + + if (_storage != null) + { + item = await GetPackageViaStorageAsync(packageItem, cancellationToken); + } + + if (item == null) + { + item = await GetPackageViaHttpAsync(packageItem, timestamp, item, cancellationToken); + } + + _logger.LogInformation( + "Finished creating package catalog item for {Id} {Version}", + packageItem.PackageId, + packageItem.PackageNormalizedVersion); + + return item; + } + + private async Task GetPackageViaStorageAsync( + FeedPackageDetails packageItem, + CancellationToken cancellationToken) + { + PackageCatalogItem item = null; + var packageId = packageItem.PackageId.ToLowerInvariant(); + var packageNormalizedVersion = packageItem.PackageNormalizedVersion.ToLowerInvariant(); + var packageFileName = PackageUtility.GetPackageFileName(packageId, packageNormalizedVersion); + var blobUri = _storage.ResolveUri(packageFileName); + var blob = await _storage.GetCloudBlockBlobReferenceAsync(blobUri); + + if (!await blob.ExistsAsync(cancellationToken)) + { + _telemetryService.TrackMetric( + TelemetryConstants.NonExistentBlob, + metric: 1, + properties: GetProperties(packageId, packageNormalizedVersion, blob)); + + return item; + } + + using (_telemetryService.TrackDuration( + TelemetryConstants.PackageBlobReadSeconds, + GetProperties(packageId, packageNormalizedVersion, blob: null))) + { + await blob.FetchAttributesAsync(cancellationToken); + + string packageHash = null; + var etag = blob.ETag; + + var metadata = await blob.GetMetadataAsync(cancellationToken); + + if (metadata.TryGetValue(Constants.Sha512, out packageHash)) + { + using (var stream = await blob.GetStreamAsync(cancellationToken)) + { + item = Utils.CreateCatalogItem( + packageItem.ContentUri.ToString(), + stream, + packageItem.CreatedDate, + packageItem.LastEditedDate, + packageItem.PublishedDate, + licenseNames: null, + licenseReportUrl: null, + packageHash: packageHash, + deprecationItem: packageItem.DeprecationInfo, + vulnerabilities: packageItem.VulnerabilityInfo); + + if (item == null) + { + _logger.LogWarning("Unable to extract metadata from: {PackageDetailsContentUri}", packageItem.ContentUri); + } + } + + if (item != null) + { + // Since obtaining the ETag the first time, it's possible (though unlikely) that the blob may + // have changed. Although reading a blob with a single GET request should return the whole + // blob in a consistent state, we're reading the blob using ZipArchive and a seekable stream, + // which results in many GET requests. To guard against the blob having changed since we + // obtained the package hash, we check the ETag one more time. If this check fails, we'll + // fallback to using a single HTTP GET request. + await blob.FetchAttributesAsync(cancellationToken); + + if (etag != blob.ETag) + { + item = null; + + _telemetryService.TrackMetric( + TelemetryConstants.BlobModified, + metric: 1, + properties: GetProperties(packageId, packageNormalizedVersion, blob)); + } + } + } + else + { + _telemetryService.TrackMetric( + TelemetryConstants.NonExistentPackageHash, + metric: 1, + properties: GetProperties(packageId, packageNormalizedVersion, blob)); + } + } + + return item; + } + + private async Task GetPackageViaHttpAsync(FeedPackageDetails packageItem, DateTime timestamp, PackageCatalogItem item, CancellationToken cancellationToken) + { + // When downloading the package binary, add a query string parameter + // that corresponds to the operation's timestamp. + // This query string will ensure the package is not cached + // (e.g. on the CDN) and returns the "latest and greatest" package metadata. + var packageUri = Utilities.GetNugetCacheBustingUri(packageItem.ContentUri, timestamp.ToString("O")); + HttpResponseMessage response = null; + + try + { + using (_telemetryService.TrackDuration( + TelemetryConstants.PackageDownloadSeconds, + GetProperties(packageItem, blob: null))) + { + response = await _httpClient.GetAsync(packageUri, cancellationToken); + } + } + catch (TaskCanceledException tce) + { + // If the HTTP request timed out, a TaskCanceledException will be thrown. + throw new HttpClientTimeoutException($"HttpClient request timed out in {nameof(PackageCatalogItemCreator.GetPackageViaHttpAsync)}.", tce); + } + + if (response.IsSuccessStatusCode) + { + using (var stream = await response.Content.ReadAsStreamAsync()) + { + item = Utils.CreateCatalogItem( + packageItem.ContentUri.ToString(), + stream, + packageItem.CreatedDate, + packageItem.LastEditedDate, + packageItem.PublishedDate, + deprecationItem: packageItem.DeprecationInfo, + vulnerabilities: packageItem.VulnerabilityInfo); + + if (item == null) + { + _logger.LogWarning("Unable to extract metadata from: {PackageDetailsContentUri}", packageItem.ContentUri); + } + } + } + else + { + if (response.StatusCode == System.Net.HttpStatusCode.NotFound) + { + // the feed is out of sync with the actual package storage - if we don't have the package there is nothing to be done we might as well move onto the next package + _logger?.LogWarning("Unable to download: {PackageDetailsContentUri}. Http status: {HttpStatusCode}", packageItem.ContentUri, response.StatusCode); + } + else + { + // this should trigger a restart - of this program - and not move the cursor forward + _logger?.LogError("Unable to download: {PackageDetailsContentUri}. Http status: {HttpStatusCode}", packageItem.ContentUri, response.StatusCode); + throw new Exception( + $"Unable to download: {packageItem.ContentUri} http status: {response.StatusCode}"); + } + } + + return item; + } + + private static Dictionary GetProperties(FeedPackageDetails packageItem, ICloudBlockBlob blob) + { + return GetProperties( + packageItem.PackageId.ToLowerInvariant(), + packageItem.PackageNormalizedVersion.ToLowerInvariant(), + blob); + } + + private static Dictionary GetProperties(string packageId, string packageNormalizedVersion, ICloudBlockBlob blob) + { + var properties = new Dictionary() + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, packageNormalizedVersion } + }; + + if (blob != null) + { + properties.Add(TelemetryConstants.Uri, blob.Uri.AbsoluteUri); + } + + return properties; + } + } +} \ No newline at end of file diff --git a/src/Catalog/PackageDeprecationItem.cs b/src/Catalog/PackageDeprecationItem.cs new file mode 100644 index 000000000..f3d611d27 --- /dev/null +++ b/src/Catalog/PackageDeprecationItem.cs @@ -0,0 +1,61 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace NuGet.Services.Metadata.Catalog +{ + public class PackageDeprecationItem + { + /// The list of reasons a package was deprecated. + /// An additional message associated with a package, if one exists. + /// + /// The ID of a package that can be used alternatively. Must be specified if is specified. + /// + /// + /// A string representing the version range of a package that can be used alternatively. Must be specified if is specified. + /// + public PackageDeprecationItem( + IReadOnlyList reasons, + string message, + string alternatePackageId, + string alternatePackageRange) + { + if (reasons == null) + { + throw new ArgumentNullException(nameof(reasons)); + } + + if (!reasons.Any()) + { + throw new ArgumentException(nameof(reasons)); + } + + Reasons = reasons; + Message = message; + AlternatePackageId = alternatePackageId; + AlternatePackageRange = alternatePackageRange; + + if (AlternatePackageId == null && AlternatePackageRange != null) + { + throw new ArgumentException( + "Cannot specify an alternate package version range if an alternate package ID is not provided.", + nameof(AlternatePackageRange)); + } + + if (AlternatePackageId != null && AlternatePackageRange == null) + { + throw new ArgumentException( + "Cannot specify an alternate package ID if an alternate package version range is not provided.", + nameof(AlternatePackageId)); + } + } + + public IReadOnlyList Reasons { get; } + public string Message { get; } + public string AlternatePackageId { get; } + public string AlternatePackageRange { get; } + } +} diff --git a/src/Catalog/PackageDownloader.cs b/src/Catalog/PackageDownloader.cs new file mode 100644 index 000000000..8fc9bb11c --- /dev/null +++ b/src/Catalog/PackageDownloader.cs @@ -0,0 +1,96 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog +{ + public class PackageDownloader + { + private const int BufferSize = 80 * 1024; + + private readonly HttpClient _httpClient; + private readonly ILogger _logger; + + public PackageDownloader(HttpClient httpClient, ILogger logger) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task DownloadAsync(Uri packageUri, CancellationToken cancellationToken) + { + _logger.LogInformation("Attempting to download package from {PackageUri}...", packageUri); + + Stream packageStream = null; + var stopwatch = Stopwatch.StartNew(); + + try + { + // Download the package from the network to a temporary file. + using (var response = await _httpClient.GetAsync(packageUri, HttpCompletionOption.ResponseHeadersRead)) + { + _logger.LogInformation( + "Received response {StatusCode}: {ReasonPhrase} of type {ContentType} for request {PackageUri}", + response.StatusCode, + response.ReasonPhrase, + response.Content.Headers.ContentType, + packageUri); + + if (response.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + if (response.StatusCode != HttpStatusCode.OK) + { + throw new InvalidOperationException($"Expected status code {HttpStatusCode.OK} for package download, actual: {response.StatusCode}"); + } + + using (var networkStream = await response.Content.ReadAsStreamAsync()) + { + packageStream = new FileStream( + Path.GetTempFileName(), + FileMode.Create, + FileAccess.ReadWrite, + FileShare.None, + BufferSize, + FileOptions.DeleteOnClose | FileOptions.Asynchronous); + + await networkStream.CopyToAsync(packageStream, BufferSize, cancellationToken); + } + } + + packageStream.Position = 0; + + stopwatch.Stop(); + + _logger.LogInformation( + "Downloaded {PackageSizeInBytes} bytes in {DownloadElapsedTime} seconds for request {PackageUri}", + packageStream.Length, + stopwatch.Elapsed.TotalSeconds, + packageUri); + + return packageStream; + } + catch (Exception e) + { + _logger.LogError( + (EventId)0, + e, + "Exception thrown when trying to download package from {PackageUri}", + packageUri); + + packageStream?.Dispose(); + + throw; + } + } + } +} diff --git a/src/Catalog/PackageEntry.cs b/src/Catalog/PackageEntry.cs new file mode 100644 index 000000000..8ab43831b --- /dev/null +++ b/src/Catalog/PackageEntry.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO.Compression; +using Newtonsoft.Json; + +namespace NuGet.Services.Metadata.Catalog +{ + public class PackageEntry + { + public PackageEntry() + { + } + + public PackageEntry(ZipArchiveEntry zipArchiveEntry) + { + if (zipArchiveEntry == null) + { + throw new ArgumentNullException(nameof(zipArchiveEntry)); + } + + FullName = zipArchiveEntry.FullName; + Name = zipArchiveEntry.Name; + Length = zipArchiveEntry.Length; + CompressedLength = zipArchiveEntry.CompressedLength; + } + + [JsonProperty("fullName")] + public string FullName { get; set; } + + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("length")] + public long Length { get; set; } + + [JsonProperty("compressedLength")] + public long CompressedLength { get; set; } + } +} \ No newline at end of file diff --git a/src/Catalog/PackageVulnerabilityItem.cs b/src/Catalog/PackageVulnerabilityItem.cs new file mode 100644 index 000000000..c6d98eda3 --- /dev/null +++ b/src/Catalog/PackageVulnerabilityItem.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog +{ + public class PackageVulnerabilityItem + { + /// GitHub Database Key - part of unique id in json + /// URL containing advisory details + /// Severity of vulnerability advisory 0-3, 0 is most critical + public PackageVulnerabilityItem( + string gitHubDatabaseKey, + string advisoryUrl, + string severity) + { + GitHubDatabaseKey = gitHubDatabaseKey; + AdvisoryUrl = advisoryUrl; + Severity = severity; + } + + public string GitHubDatabaseKey { get; } + public string AdvisoryUrl { get; } + public string Severity { get; } + } +} diff --git a/src/Catalog/PackagesContainerCatalogProcessor.cs b/src/Catalog/PackagesContainerCatalogProcessor.cs new file mode 100644 index 000000000..22dad13fe --- /dev/null +++ b/src/Catalog/PackagesContainerCatalogProcessor.cs @@ -0,0 +1,160 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net; +using System.Net.Sockets; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog +{ + /// + /// A processor that does work on a package's blob in the packages container. + /// + public class PackagesContainerCatalogProcessor : ICatalogIndexProcessor + { + private const int MaximumPackageProcessingAttempts = 5; + + private readonly CloudBlobContainer _container; + private readonly IPackagesContainerHandler _handler; + private readonly ITelemetryService _telemetryService; + private readonly ILogger _logger; + + /// + /// Create a new packages container processor. + /// + /// The reference to the packages container. + /// The handler that will be run on blobs in the packages container. + /// The service used to track events. + /// + public PackagesContainerCatalogProcessor( + CloudBlobContainer container, + IPackagesContainerHandler handler, + ITelemetryService telemetryService, + ILogger logger) + { + _container = container ?? throw new ArgumentNullException(nameof(container)); + _handler = handler ?? throw new ArgumentNullException(nameof(handler)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ProcessCatalogIndexEntryAsync(CatalogIndexEntry catalogEntry) + { + try + { + var rawBlob = _container.GetBlockBlobReference(BuildPackageFileName(catalogEntry)); + var blob = new AzureCloudBlockBlob(rawBlob); + + for (int i = 0; i < MaximumPackageProcessingAttempts; i++) + { + try + { + await _handler.ProcessPackageAsync(catalogEntry, blob); + return; + } + catch (Exception e) when (IsRetryableException(e)) + { + _logger.LogWarning( + 0, + e, + "Processing package {PackageId} {PackageVersion} failed due to an uncaught exception. " + + $"Attempt {{Attempt}} of {MaximumPackageProcessingAttempts}", + catalogEntry.Id, + catalogEntry.Version, + i + 1); + } + } + + _telemetryService.TrackHandlerFailedToProcessPackage(_handler, catalogEntry.Id, catalogEntry.Version); + + _logger.LogError( + $"Failed to process package {{PackageId}} {{PackageVersion}} after {MaximumPackageProcessingAttempts} attempts", + catalogEntry.Id, + catalogEntry.Version); + } + catch (StorageException e) when (IsPackageDoesNotExistException(e)) + { + // This indicates a discrepancy between v2 and v3 APIs that should be caught by + // the monitoring job. No need to track this handler failure. + _logger.LogError( + "Package {PackageId} {PackageVersion} is missing from the packages container!", + catalogEntry.Id, + catalogEntry.Version); + } + catch (Exception e) + { + _telemetryService.TrackHandlerFailedToProcessPackage(_handler, catalogEntry.Id, catalogEntry.Version); + + _logger.LogError( + 0, + e, + "Could not process package {PackageId} {PackageVersion}", + catalogEntry.Id, + catalogEntry.Version); + } + } + + private string BuildPackageFileName(CatalogIndexEntry packageEntry) + { + var packageId = packageEntry.Id.ToLowerInvariant(); + var packageVersion = packageEntry.Version.ToNormalizedString().ToLowerInvariant(); + + return $"{packageId}.{packageVersion}.nupkg"; + } + + private static bool IsRetryableException(Exception exception) + { + // Retry on HTTP Client timeouts. + if (exception is TaskCanceledException) + { + return true; + } + + // Retry if updating a blob fails due to an incorrect ETag. + if (exception is StorageException storageException && IsPackageWasModifiedException(storageException)) + { + return true; + } + + do + { + // Retry on time out exceptions. + if (exception is TimeoutException) + { + return true; + } + + if (exception is WebException we && we.Status == WebExceptionStatus.Timeout) + { + return true; + } + + // Retry if the host forcibly closes the connection. + if (exception is SocketException) + { + return true; + } + + exception = exception.InnerException; + } + while (exception != null); + + return false; + } + + private static bool IsPackageDoesNotExistException(StorageException exception) + { + return exception?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.NotFound; + } + + private static bool IsPackageWasModifiedException(StorageException exception) + { + return exception?.RequestInformation?.HttpStatusCode == (int?)HttpStatusCode.PreconditionFailed; + } + } +} diff --git a/src/Catalog/Persistence/AzureCloudBlockBlob.cs b/src/Catalog/Persistence/AzureCloudBlockBlob.cs new file mode 100644 index 000000000..90029a551 --- /dev/null +++ b/src/Catalog/Persistence/AzureCloudBlockBlob.cs @@ -0,0 +1,63 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public sealed class AzureCloudBlockBlob : ICloudBlockBlob + { + private readonly CloudBlockBlob _blob; + + /// + /// The Base64 encoded MD5 hash of the blob's content + /// + public string ContentMD5 + { + get => _blob.Properties.ContentMD5; + set => _blob.Properties.ContentMD5 = value; + } + + public string ETag => _blob.Properties.ETag; + public long Length => _blob.Properties.Length; + public Uri Uri => _blob.Uri; + + public AzureCloudBlockBlob(CloudBlockBlob blob) + { + _blob = blob ?? throw new ArgumentNullException(nameof(blob)); + } + + public async Task ExistsAsync(CancellationToken cancellationToken) + { + return await _blob.ExistsAsync(); + } + + public async Task FetchAttributesAsync(CancellationToken cancellationToken) + { + await _blob.FetchAttributesAsync(cancellationToken); + } + + public async Task> GetMetadataAsync(CancellationToken cancellationToken) + { + return await Task.FromResult>( + new ReadOnlyDictionary(_blob.Metadata)); + } + + public async Task GetStreamAsync(CancellationToken cancellationToken) + { + return await _blob.OpenReadAsync(cancellationToken); + } + + public async Task SetPropertiesAsync(AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext) + { + await _blob.SetPropertiesAsync(accessCondition, options, operationContext); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/AzureStorage.cs b/src/Catalog/Persistence/AzureStorage.cs new file mode 100644 index 000000000..3d30e5a03 --- /dev/null +++ b/src/Catalog/Persistence/AzureStorage.cs @@ -0,0 +1,525 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Auth; +using Microsoft.WindowsAzure.Storage.Blob; +using Microsoft.WindowsAzure.Storage.DataMovement; +using Microsoft.WindowsAzure.Storage.RetryPolicies; +using NuGet.Protocol; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public class AzureStorage : Storage, IAzureStorage + { + private readonly bool _compressContent; + private readonly IThrottle _throttle; + private readonly ICloudBlobDirectory _directory; + private readonly bool _useServerSideCopy; + + public const string Sha512HashAlgorithmId = "SHA512"; + public static readonly TimeSpan DefaultServerTimeout = TimeSpan.FromSeconds(30); + public static readonly TimeSpan DefaultMaxExecutionTime = TimeSpan.FromMinutes(10); + + public AzureStorage( + CloudStorageAccount account, + string containerName, + string path, + Uri baseAddress, + TimeSpan maxExecutionTime, + TimeSpan serverTimeout, + bool useServerSideCopy, + bool compressContent, + bool verbose, + bool initializeContainer, + IThrottle throttle) : this( + new CloudBlobDirectoryWrapper(account.CreateCloudBlobClient().GetContainerReference(containerName).GetDirectoryReference(path)), + baseAddress, + maxExecutionTime, + serverTimeout, + initializeContainer) + { + _useServerSideCopy = useServerSideCopy; + _compressContent = compressContent; + _throttle = throttle ?? NullThrottle.Instance; + Verbose = verbose; + } + + public AzureStorage( + Uri storageBaseUri, + TimeSpan maxExecutionTime, + TimeSpan serverTimeout, + bool useServerSideCopy, + bool compressContent, + bool verbose, + IThrottle throttle) + : this(GetCloudBlobDirectoryUri(storageBaseUri), storageBaseUri, maxExecutionTime, serverTimeout, initializeContainer: false) + { + _useServerSideCopy = useServerSideCopy; + _compressContent = compressContent; + _throttle = throttle ?? NullThrottle.Instance; + Verbose = verbose; + } + + private static ICloudBlobDirectory GetCloudBlobDirectoryUri(Uri storageBaseUri) + { + if (storageBaseUri.AbsoluteUri.Contains('%')) + { + // Later in the code for the sake of simplicity wrong things are done with URL that + // can explode when URL is specially crafted with certain URL-encoded characters. + // Since it is URL for our storage root where we know that we don't use anything + // that requires URL-encoding, we'll just throw here just in case, to keep code + // below simple. + throw new ArgumentException("Storage URL cannot contain URL-encoded characters"); + } + + var pathSegments = storageBaseUri.AbsolutePath.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + if (pathSegments.Length < 1) + { + throw new ArgumentException("Storage URL must contain some path"); + } + + var anonymousCredentials = new StorageCredentials(); + var blobEndpoint = new Uri(storageBaseUri.GetComponents(UriComponents.SchemeAndServer, UriFormat.Unescaped)); + var storageAccount = new CloudStorageAccount(anonymousCredentials, blobEndpoint, queueEndpoint: null, tableEndpoint: null, fileEndpoint: null); + var containerName = pathSegments[0]; + var pathInContainer = string.Join("/", pathSegments.Skip(1)); + var container = storageAccount.CreateCloudBlobClient().GetContainerReference(containerName); + return new CloudBlobDirectoryWrapper(container.GetDirectoryReference(pathInContainer)); + } + + public AzureStorage( + ICloudBlobDirectory directory, + Uri baseAddress, + TimeSpan maxExecutionTime, + TimeSpan serverTimeout, + bool initializeContainer) : base( + baseAddress ?? GetDirectoryUri(directory)) + { + _directory = directory; + + // Unless overridden at the level of a single API call, these options will apply to all service calls that + // use BlobRequestOptions. + _directory.ServiceClient.DefaultRequestOptions = new BlobRequestOptions() + { + ServerTimeout = serverTimeout, + MaximumExecutionTime = maxExecutionTime, + RetryPolicy = new ExponentialRetry() + }; + + if (initializeContainer) + { + if (_directory.Container.CreateIfNotExists()) + { + _directory.Container.SetPermissions(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob }); + + if (Verbose) + { + Trace.WriteLine(string.Format("Created '{0}' public container", _directory.Container.Name)); + } + } + } + } + + public override async Task GetOptimisticConcurrencyControlTokenAsync( + Uri resourceUri, + CancellationToken cancellationToken) + { + if (resourceUri == null) + { + throw new ArgumentNullException(nameof(resourceUri)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + string blobName = GetName(resourceUri); + CloudBlockBlob blob = GetBlockBlobReference(blobName); + + await blob.FetchAttributesAsync(cancellationToken); + + return new OptimisticConcurrencyControlToken(blob.Properties.ETag); + } + + private static Uri GetDirectoryUri(ICloudBlobDirectory directory) + { + Uri uri = new UriBuilder(directory.Uri) + { + Scheme = "http", + Port = 80 + }.Uri; + + return uri; + } + + //Blob exists + public override bool Exists(string fileName) + { + Uri packageRegistrationUri = ResolveUri(fileName); + string blobName = GetName(packageRegistrationUri); + + CloudBlockBlob blob = GetBlockBlobReference(blobName); + + if (blob.Exists()) + { + return true; + } + if (Verbose) + { + Trace.WriteLine(string.Format("The blob {0} does not exist.", packageRegistrationUri)); + } + return false; + } + + public override async Task> ListAsync(CancellationToken cancellationToken) + { + var files = await _directory.ListBlobsAsync(cancellationToken); + + return files.Select(GetStorageListItem).AsEnumerable(); + } + + private StorageListItem GetStorageListItem(IListBlobItem listBlobItem) + { + var lastModified = (listBlobItem as CloudBlockBlob)?.Properties.LastModified?.UtcDateTime; + + return new StorageListItem(listBlobItem.Uri, lastModified); + } + + protected override async Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + var azureDestinationStorage = destinationStorage as AzureStorage; + + if (azureDestinationStorage == null) + { + throw new NotImplementedException("Copying is only supported from Azure storage to Azure storage."); + } + + string sourceName = GetName(sourceUri); + string destinationName = azureDestinationStorage.GetName(destinationUri); + + CloudBlockBlob sourceBlob = GetBlockBlobReference(sourceName); + CloudBlockBlob destinationBlob = azureDestinationStorage.GetBlockBlobReference(destinationName); + + var context = new SingleTransferContext(); + + if (destinationProperties?.Count > 0) + { + context.SetAttributesCallback = new SetAttributesCallback((destination) => + { + var blob = (CloudBlockBlob)destination; + + // The copy statement copied all properties from the source blob to the destination blob; however, + // there may be required properties on destination blob, all of which may have not already existed + // on the source blob at the time of copy. + foreach (var property in destinationProperties) + { + switch (property.Key) + { + case StorageConstants.CacheControl: + blob.Properties.CacheControl = property.Value; + break; + + case StorageConstants.ContentType: + blob.Properties.ContentType = property.Value; + break; + + default: + throw new NotImplementedException($"Storage property '{property.Value}' is not supported."); + } + } + }); + } + + context.ShouldOverwriteCallback = new ShouldOverwriteCallback((source, destination) => true); + + await TransferManager.CopyAsync(sourceBlob, destinationBlob, _useServerSideCopy, options: null, context: context); + } + + protected override async Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + { + string name = GetName(resourceUri); + + CloudBlockBlob blob = GetBlockBlobReference(name); + + blob.Properties.ContentType = content.ContentType; + blob.Properties.CacheControl = content.CacheControl; + + if (_compressContent) + { + blob.Properties.ContentEncoding = "gzip"; + using (Stream stream = content.GetContentStream()) + { + MemoryStream destinationStream = new MemoryStream(); + + using (GZipStream compressionStream = new GZipStream(destinationStream, CompressionMode.Compress, true)) + { + await stream.CopyToAsync(compressionStream); + } + + destinationStream.Seek(0, SeekOrigin.Begin); + + var accessCondition = (content as StringStorageContentWithAccessCondition)?.AccessCondition; + + await blob.UploadFromStreamAsync( + destinationStream, + accessCondition, + options: null, + operationContext: null, + cancellationToken: cancellationToken); + + Trace.WriteLine(string.Format("Saved compressed blob {0} to container {1}", blob.Uri.ToString(), _directory.Container.Name)); + } + } + else + { + using (Stream stream = content.GetContentStream()) + { + await blob.UploadFromStreamAsync(stream, cancellationToken); + } + + Trace.WriteLine(string.Format("Saved uncompressed blob {0} to container {1}", blob.Uri.ToString(), _directory.Container.Name)); + } + + await TryTakeBlobSnapshotAsync(blob); + } + + /// + /// Take one snapshot only if there is not any snapshot for the specific blob + /// This will prevent the blob to be deleted by a not intended delete action + /// + /// + /// + private async Task TryTakeBlobSnapshotAsync(CloudBlockBlob blob) + { + if (blob == null) + { + //no action + return false; + } + + var stopwatch = Stopwatch.StartNew(); + + try + { + var allSnapshots = blob.Container. + ListBlobs(prefix: blob.Name, + useFlatBlobListing: true, + blobListingDetails: BlobListingDetails.Snapshots); + //the above call will return at least one blob the original + if (allSnapshots.Count() == 1) + { + var snapshot = await blob.CreateSnapshotAsync(); + stopwatch.Stop(); + Trace.WriteLine($"SnapshotCreated:milliseconds={stopwatch.ElapsedMilliseconds}:{blob.Uri.ToString()}:{snapshot.SnapshotQualifiedUri}"); + } + return true; + } + catch (StorageException storageException) + { + stopwatch.Stop(); + Trace.WriteLine($"EXCEPTION:milliseconds={stopwatch.ElapsedMilliseconds}:CreateSnapshot: Failed to take the snapshot for blob {blob.Uri.ToString()}. Exception{storageException.ToString()}"); + return false; + } + } + + protected override async Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken) + { + // the Azure SDK will treat a starting / as an absolute URL, + // while we may be working in a subdirectory of a storage container + // trim the starting slash to treat it as a relative path + string name = GetName(resourceUri).TrimStart('/'); + + CloudBlockBlob blob = GetBlockBlobReference(name); + + await _throttle.WaitAsync(); + try + { + string content; + + using (var originalStream = new MemoryStream()) + { + await blob.DownloadToStreamAsync(originalStream, cancellationToken); + + originalStream.Seek(0, SeekOrigin.Begin); + + if (blob.Properties.ContentEncoding == "gzip") + { + using (var uncompressedStream = new GZipStream(originalStream, CompressionMode.Decompress)) + { + using (var reader = new StreamReader(uncompressedStream)) + { + content = await reader.ReadToEndAsync(); + } + } + } + else + { + using (var reader = new StreamReader(originalStream)) + { + content = await reader.ReadToEndAsync(); + } + } + } + + return new StringStorageContentWithETag(content, blob.Properties.ETag); + } + catch (StorageException ex) when (ex.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.NotFound) + { + if (Verbose) + { + Trace.WriteLine(string.Format("Can't load '{0}'. Blob doesn't exist", resourceUri)); + } + + return null; + } + finally + { + _throttle.Release(); + } + } + + protected override async Task OnDeleteAsync(Uri resourceUri, DeleteRequestOptions deleteRequestOptions, CancellationToken cancellationToken) + { + string name = GetName(resourceUri); + + var accessCondition = (deleteRequestOptions as DeleteRequestOptionsWithAccessCondition)?.AccessCondition; + + CloudBlockBlob blob = GetBlockBlobReference(name); + await blob.DeleteAsync(deleteSnapshotsOption: DeleteSnapshotsOption.IncludeSnapshots, + accessCondition: accessCondition, + options: null, + operationContext: null, + cancellationToken: cancellationToken); + } + + /// + /// Returns the uri of the blob based on the Azure cloud directory + /// + /// The blob name. + /// The blob uri. + public override Uri GetUri(string name) + { + var baseUri = _directory.Uri.AbsoluteUri; + + if (baseUri.EndsWith("/")) + { + return new Uri($"{baseUri}{name}", UriKind.Absolute); + } + + return new Uri($"{baseUri}/{name}", UriKind.Absolute); + } + + public override async Task AreSynchronized(Uri firstResourceUri, Uri secondResourceUri) + { + var source = new CloudBlockBlob(firstResourceUri); + var destination = GetBlockBlobReference(GetName(secondResourceUri)); + + // For interacting with the source, we just use the same blob request options as the destination blob. + ApplyBlobRequestOptions(source); + + return await AreSynchronized(new AzureCloudBlockBlob(source), new AzureCloudBlockBlob(destination)); + } + + public async Task AreSynchronized(ICloudBlockBlob sourceBlockBlob, ICloudBlockBlob destinationBlockBlob) + { + if (await destinationBlockBlob.ExistsAsync(CancellationToken.None)) + { + if (await sourceBlockBlob.ExistsAsync(CancellationToken.None)) + { + var sourceBlobMetadata = await sourceBlockBlob.GetMetadataAsync(CancellationToken.None); + var destinationBlobMetadata = await destinationBlockBlob.GetMetadataAsync(CancellationToken.None); + if (sourceBlobMetadata == null || destinationBlobMetadata == null) + { + return false; + } + + var sourceBlobHasSha512Hash = sourceBlobMetadata.TryGetValue(Sha512HashAlgorithmId, out var sourceBlobSha512Hash); + var destinationBlobHasSha512Hash = destinationBlobMetadata.TryGetValue(Sha512HashAlgorithmId, out var destinationBlobSha512Hash); + if (!sourceBlobHasSha512Hash) + { + Trace.TraceWarning(string.Format("The source blob ({0}) doesn't have the SHA512 hash.", sourceBlockBlob.Uri.ToString())); + } + if (!destinationBlobHasSha512Hash) + { + Trace.TraceWarning(string.Format("The destination blob ({0}) doesn't have the SHA512 hash.", destinationBlockBlob.Uri.ToString())); + } + if (sourceBlobHasSha512Hash && destinationBlobHasSha512Hash) + { + if (sourceBlobSha512Hash == destinationBlobSha512Hash) + { + Trace.WriteLine(string.Format("The source blob ({0}) and destination blob ({1}) have the same SHA512 hash and are synchronized.", + sourceBlockBlob.Uri.ToString(), destinationBlockBlob.Uri.ToString())); + return true; + } + + // The SHA512 hash between the source and destination blob should be always same. + Trace.TraceWarning(string.Format("The source blob ({0}) and destination blob ({1}) have the different SHA512 hash and are not synchronized. " + + "The source blob hash is {2} while the destination blob hash is {3}", + sourceBlockBlob.Uri.ToString(), destinationBlockBlob.Uri.ToString(), sourceBlobSha512Hash, destinationBlobSha512Hash)); + } + + return false; + } + return true; + } + return !(await sourceBlockBlob.ExistsAsync(CancellationToken.None)); + } + + public async Task GetCloudBlockBlobReferenceAsync(Uri blobUri) + { + string blobName = GetName(blobUri); + CloudBlockBlob blob = GetBlockBlobReference(blobName); + var blobExists = await blob.ExistsAsync(); + + if (Verbose && !blobExists) + { + Trace.WriteLine($"The blob {blobUri.AbsoluteUri} does not exist."); + } + + return new AzureCloudBlockBlob(blob); + } + + public async Task HasPropertiesAsync(Uri blobUri, string contentType, string cacheControl) + { + var blobName = GetName(blobUri); + var blob = GetBlockBlobReference(blobName); + + if (await blob.ExistsAsync()) + { + await blob.FetchAttributesAsync(); + + return string.Equals(blob.Properties.ContentType, contentType) + && string.Equals(blob.Properties.CacheControl, cacheControl); + } + + return false; + } + + private CloudBlockBlob GetBlockBlobReference(string blobName) + { + var blob = _directory.GetBlockBlobReference(blobName); + + ApplyBlobRequestOptions(blob); + + return blob; + } + + private void ApplyBlobRequestOptions(CloudBlockBlob blob) + { + blob.ServiceClient.DefaultRequestOptions = _directory.ServiceClient.DefaultRequestOptions; + } + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/AzureStorageFactory.cs b/src/Catalog/Persistence/AzureStorageFactory.cs new file mode 100644 index 000000000..8f0315dc9 --- /dev/null +++ b/src/Catalog/Persistence/AzureStorageFactory.cs @@ -0,0 +1,110 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.WindowsAzure.Storage; +using NuGet.Protocol; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public class AzureStorageFactory : StorageFactory + { + private readonly CloudStorageAccount _account; + private readonly string _containerName; + private readonly string _path; + private readonly Uri _differentBaseAddress = null; + private readonly TimeSpan _maxExecutionTime; + private readonly TimeSpan _serverTimeout; + private readonly bool _useServerSideCopy; + private readonly bool _initializeContainer; + + public AzureStorageFactory( + CloudStorageAccount account, + string containerName, + TimeSpan maxExecutionTime, + TimeSpan serverTimeout, + string path, + Uri baseAddress, + bool useServerSideCopy, + bool compressContent, + bool verbose, + bool initializeContainer, + IThrottle throttle) : base(throttle) + { + _account = account; + _containerName = containerName; + _path = null; + _maxExecutionTime = maxExecutionTime; + _serverTimeout = serverTimeout; + _useServerSideCopy = useServerSideCopy; + _initializeContainer = initializeContainer; + + if (path != null) + { + _path = path.Trim('/') + '/'; + } + + _differentBaseAddress = baseAddress; + + var blobEndpointBuilder = new UriBuilder(account.BlobEndpoint) + { + Scheme = "http", // Convert base address to http. 'https' can be used for communication but is not part of the names. + Port = 80 + }; + + if (baseAddress == null) + { + BaseAddress = new Uri(blobEndpointBuilder.Uri, containerName + "/" + _path ?? string.Empty); + } + else + { + Uri newAddress = baseAddress; + + if (path != null) + { + newAddress = new Uri(baseAddress, path + "/"); + } + + BaseAddress = newAddress; + } + + // Beautify the destination URL. + blobEndpointBuilder.Scheme = "https"; + blobEndpointBuilder.Port = -1; + + DestinationAddress = new Uri(blobEndpointBuilder.Uri, containerName + "/" + _path ?? string.Empty); + + CompressContent = compressContent; + Verbose = verbose; + } + + public bool CompressContent { get; } + + public override Storage Create(string name = null) + { + string path = (_path == null) ? name : _path + name; + + path = (name == null) ? (_path == null ? String.Empty : _path.Trim('/')) : path; + + Uri newBase = _differentBaseAddress; + + if (newBase != null && !string.IsNullOrEmpty(name)) + { + newBase = new Uri(_differentBaseAddress, name + "/"); + } + + return new AzureStorage( + _account, + _containerName, + path, + newBase, + _maxExecutionTime, + _serverTimeout, + _useServerSideCopy, + CompressContent, + Verbose, + _initializeContainer, + Throttle); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/ByteArrayStorageContent.cs b/src/Catalog/Persistence/ByteArrayStorageContent.cs new file mode 100644 index 000000000..9ab5fe2c6 --- /dev/null +++ b/src/Catalog/Persistence/ByteArrayStorageContent.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.IO; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public class ByteArrayStorageContent : StorageContent + { + public ByteArrayStorageContent(byte[] content, string contentType = "", string cacheControl = "") + { + Content = content; + ContentType = contentType; + CacheControl = cacheControl; + } + + public byte[] Content { get; set; } + + public override Stream GetContentStream() + { + if (Content == null) + { + return null; + } + + return new MemoryStream(Content); + } + } +} diff --git a/src/Catalog/Persistence/CloudBlobDirectoryWrapper.cs b/src/Catalog/Persistence/CloudBlobDirectoryWrapper.cs new file mode 100644 index 000000000..6fa059147 --- /dev/null +++ b/src/Catalog/Persistence/CloudBlobDirectoryWrapper.cs @@ -0,0 +1,35 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using System.Collections.Generic; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public class CloudBlobDirectoryWrapper : ICloudBlobDirectory + { + private readonly CloudBlobDirectory _blobDirectory; + + public ICloudBlockBlobClient ServiceClient => new CloudBlockBlobClientWrapper(_blobDirectory.ServiceClient); + public CloudBlobContainer Container => _blobDirectory.Container; + public Uri Uri => _blobDirectory.Uri; + + public CloudBlobDirectoryWrapper(CloudBlobDirectory blobDirectory) + { + _blobDirectory = blobDirectory ?? throw new ArgumentNullException(nameof(blobDirectory)); + } + + public CloudBlockBlob GetBlockBlobReference(string blobName) + { + return _blobDirectory.GetBlockBlobReference(blobName); + } + + public async Task> ListBlobsAsync(CancellationToken cancellationToken) + { + return await _blobDirectory.ListBlobsAsync(cancellationToken); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/CloudBlockBlobClientWrapper.cs b/src/Catalog/Persistence/CloudBlockBlobClientWrapper.cs new file mode 100644 index 000000000..cf5ddb32b --- /dev/null +++ b/src/Catalog/Persistence/CloudBlockBlobClientWrapper.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.WindowsAzure.Storage.Blob; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public class CloudBlockBlobClientWrapper : ICloudBlockBlobClient + { + private readonly CloudBlobClient _blobClient; + + public BlobRequestOptions DefaultRequestOptions + { + get => _blobClient.DefaultRequestOptions; + set => _blobClient.DefaultRequestOptions = value; + } + + public CloudBlockBlobClientWrapper(CloudBlobClient blobClient) + { + _blobClient = blobClient; + } + } +} diff --git a/src/Catalog/Persistence/DeleteRequestOptions.cs b/src/Catalog/Persistence/DeleteRequestOptions.cs new file mode 100644 index 000000000..47136d5bb --- /dev/null +++ b/src/Catalog/Persistence/DeleteRequestOptions.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.WindowsAzure.Storage; +using System; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public abstract class DeleteRequestOptions + { + } + + /// + /// Allows specifying an for use by in a operation. + /// + public class DeleteRequestOptionsWithAccessCondition : DeleteRequestOptions + { + public DeleteRequestOptionsWithAccessCondition(AccessCondition accessCondition) + { + AccessCondition = accessCondition ?? throw new ArgumentNullException(nameof(accessCondition)); + } + + public AccessCondition AccessCondition { get; } + } +} diff --git a/src/Catalog/Persistence/FileStorage.cs b/src/Catalog/Persistence/FileStorage.cs new file mode 100644 index 000000000..bf1852adb --- /dev/null +++ b/src/Catalog/Persistence/FileStorage.cs @@ -0,0 +1,163 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public class FileStorage : Storage + { + public FileStorage(string baseAddress, string path, bool verbose) + : this(new Uri(baseAddress), path, verbose) + { + } + + public FileStorage(Uri baseAddress, string path, bool verbose) + : base(baseAddress) + { + Path = path; + + DirectoryInfo directoryInfo = new DirectoryInfo(path); + if (!directoryInfo.Exists) + { + directoryInfo.Create(); + } + + Verbose = verbose; + } + + //File exists + public override bool Exists(string fileName) + { + return File.Exists(fileName); + } + + public override Task> ListAsync(CancellationToken cancellationToken) + { + DirectoryInfo directoryInfo = new DirectoryInfo(Path); + var files = directoryInfo.GetFiles("*", SearchOption.AllDirectories) + .Select(file => + new StorageListItem(GetUri(file.FullName.Replace(Path, string.Empty)), file.LastWriteTimeUtc)); + + return Task.FromResult(files.AsEnumerable()); + } + + public string Path + { + get; + set; + } + + protected override Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + protected override async Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + { + TraceMethod("SAVE", resourceUri); + + string name = GetName(resourceUri); + + string path = Path.Trim('\\') + '\\'; + + string[] t = name.Split('/'); + + name = t[t.Length - 1]; + + if (t.Length > 1) + { + for (int i = 0; i < t.Length - 1; i++) + { + string folder = t[i]; + + if (folder != string.Empty) + { + if (!(new DirectoryInfo(path + folder)).Exists) + { + DirectoryInfo directoryInfo = new DirectoryInfo(path); + directoryInfo.CreateSubdirectory(folder); + } + + path = path + folder + '\\'; + } + } + } + + using (FileStream stream = File.Create(path + name)) + { + await content.GetContentStream().CopyToAsync(stream, 4096, cancellationToken); + } + } + + protected override async Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken) + { + string name = GetName(resourceUri); + + string path = Path.Trim('\\') + '\\'; + + string folder = string.Empty; + + string[] t = name.Split('/'); + if (t.Length == 2) + { + folder = t[0]; + name = t[1]; + } + + if (folder != string.Empty) + { + folder = folder + '\\'; + } + + string filename = path + folder + name; + + FileInfo fileInfo = new FileInfo(filename); + if (fileInfo.Exists) + { + return await Task.Run(() => { return new StreamStorageContent(fileInfo.OpenRead()); }, cancellationToken); + } + + return null; + } + + protected override async Task OnDeleteAsync(Uri resourceUri, DeleteRequestOptions deleteRequestOptions, CancellationToken cancellationToken) + { + string name = GetName(resourceUri); + + string path = Path.Trim('\\') + '\\'; + + string folder = string.Empty; + + string[] t = name.Split('/'); + if (t.Length == 2) + { + folder = t[0]; + name = t[1]; + } + + if (folder != string.Empty) + { + folder = folder + '\\'; + } + + string filename = path + folder + name; + + FileInfo fileInfo = new FileInfo(filename); + if (fileInfo.Exists) + { + await Task.Run(() => { fileInfo.Delete(); }, cancellationToken); + } + } + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/FileStorageFactory.cs b/src/Catalog/Persistence/FileStorageFactory.cs new file mode 100644 index 000000000..be8296988 --- /dev/null +++ b/src/Catalog/Persistence/FileStorageFactory.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public class FileStorageFactory : StorageFactory + { + private readonly string _path; + + public FileStorageFactory(Uri baseAddress, string path, bool verbose) + { + BaseAddress = new Uri(baseAddress.ToString().TrimEnd('/') + '/'); + _path = path.TrimEnd('\\') + '\\'; + + Verbose = verbose; + } + + public override Storage Create(string name = null) + { + string fileSystemPath = (name == null) ? _path.Trim('\\') : _path + name; + string uriPath = name ?? string.Empty; + + return new FileStorage(BaseAddress + uriPath, fileSystemPath, Verbose); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/IAzureStorage.cs b/src/Catalog/Persistence/IAzureStorage.cs new file mode 100644 index 000000000..96c4e1733 --- /dev/null +++ b/src/Catalog/Persistence/IAzureStorage.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public interface IAzureStorage : IStorage + { + Task GetCloudBlockBlobReferenceAsync(Uri uri); + Task HasPropertiesAsync(Uri blobUri, string contentType, string cacheControl); + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/ICloudBlobDirectory.cs b/src/Catalog/Persistence/ICloudBlobDirectory.cs new file mode 100644 index 000000000..f008f4d94 --- /dev/null +++ b/src/Catalog/Persistence/ICloudBlobDirectory.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using System.Collections.Generic; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public interface ICloudBlobDirectory + { + ICloudBlockBlobClient ServiceClient { get; } + CloudBlobContainer Container { get; } + Uri Uri { get; } + + CloudBlockBlob GetBlockBlobReference(string blobName); + Task> ListBlobsAsync(CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/ICloudBlockBlob.cs b/src/Catalog/Persistence/ICloudBlockBlob.cs new file mode 100644 index 000000000..1e17a7b0d --- /dev/null +++ b/src/Catalog/Persistence/ICloudBlockBlob.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public interface ICloudBlockBlob + { + string ContentMD5 { get; set; } + string ETag { get; } + long Length { get; } + Uri Uri { get; } + + Task ExistsAsync(CancellationToken cancellationToken); + Task FetchAttributesAsync(CancellationToken cancellationToken); + Task> GetMetadataAsync(CancellationToken cancellationToken); + Task GetStreamAsync(CancellationToken cancellationToken); + Task SetPropertiesAsync(AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext); + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/ICloudBlockBlobClient.cs b/src/Catalog/Persistence/ICloudBlockBlobClient.cs new file mode 100644 index 000000000..ca307b169 --- /dev/null +++ b/src/Catalog/Persistence/ICloudBlockBlobClient.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.WindowsAzure.Storage.Blob; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public interface ICloudBlockBlobClient + { + BlobRequestOptions DefaultRequestOptions { get; set; } + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/IStorage.cs b/src/Catalog/Persistence/IStorage.cs new file mode 100644 index 000000000..6271b1fd8 --- /dev/null +++ b/src/Catalog/Persistence/IStorage.cs @@ -0,0 +1,31 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public interface IStorage + { + Uri BaseAddress { get; } + + Task CopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellation); + Task DeleteAsync(Uri resourceUri, CancellationToken cancellationToken, DeleteRequestOptions deleteRequestOptions = null); + Task GetOptimisticConcurrencyControlTokenAsync( + Uri resourceUri, + CancellationToken cancellationToken); + Task> ListAsync(CancellationToken cancellationToken); + Task LoadAsync(Uri resourceUri, CancellationToken cancellationToken); + Task LoadStringAsync(Uri resourceUri, CancellationToken cancellationToken); + Uri ResolveUri(string relativeUri); + Task SaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/IStorageFactory.cs b/src/Catalog/Persistence/IStorageFactory.cs new file mode 100644 index 000000000..1ee7c7f4f --- /dev/null +++ b/src/Catalog/Persistence/IStorageFactory.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public interface IStorageFactory + { + Storage Create(string name = null); + Uri BaseAddress { get; } + } +} diff --git a/src/Catalog/Persistence/JTokenStorageContent.cs b/src/Catalog/Persistence/JTokenStorageContent.cs new file mode 100644 index 000000000..96a2334ea --- /dev/null +++ b/src/Catalog/Persistence/JTokenStorageContent.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.IO; +using Newtonsoft.Json.Linq; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public class JTokenStorageContent : StorageContent + { + public JTokenStorageContent(JToken content, string contentType = "", string cacheControl = "") + { + Content = content; + ContentType = contentType; + CacheControl = cacheControl; + } + + public JToken Content + { + get; + set; + } + + public override Stream GetContentStream() + { + if (Content == null) + { + return null; + } + else + { + Stream stream = new MemoryStream(); + StreamWriter writer = new StreamWriter(stream); + writer.Write(Content.ToString(Newtonsoft.Json.Formatting.None, new Newtonsoft.Json.JsonConverter[0])); + writer.Flush(); + stream.Seek(0, SeekOrigin.Begin); + return stream; + } + } + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/NamedStorageFactory.cs b/src/Catalog/Persistence/NamedStorageFactory.cs new file mode 100644 index 000000000..50ca6b025 --- /dev/null +++ b/src/Catalog/Persistence/NamedStorageFactory.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + /// + /// Appends a name to the path of another . + /// + public class NamedStorageFactory : IStorageFactory + { + private IStorageFactory _innerStorageFactory; + private string _name; + + public NamedStorageFactory(IStorageFactory inner, string name) + { + _innerStorageFactory = inner; + _name = name; + } + + public Uri BaseAddress => new Uri(_innerStorageFactory.BaseAddress, _name); + + public Storage Create(string name = null) + { + return _innerStorageFactory.Create($"{_name}/{name}"); + } + } +} diff --git a/src/Catalog/Persistence/OptimisticConcurrencyControlToken.cs b/src/Catalog/Persistence/OptimisticConcurrencyControlToken.cs new file mode 100644 index 000000000..0855ce6cd --- /dev/null +++ b/src/Catalog/Persistence/OptimisticConcurrencyControlToken.cs @@ -0,0 +1,65 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public sealed class OptimisticConcurrencyControlToken : IEquatable + { + private readonly string _token; + + public static readonly OptimisticConcurrencyControlToken Null = new OptimisticConcurrencyControlToken(token: null); + + public OptimisticConcurrencyControlToken(string token) + { + _token = token; + } + + public bool Equals(OptimisticConcurrencyControlToken other) + { + var azureToken = other as OptimisticConcurrencyControlToken; + + if (azureToken == null) + { + return false; + } + + return _token == azureToken._token; + } + + public override bool Equals(object obj) + { + return Equals(obj as OptimisticConcurrencyControlToken); + } + + public override int GetHashCode() + { + return _token.GetHashCode(); + } + + public static bool operator ==( + OptimisticConcurrencyControlToken token1, + OptimisticConcurrencyControlToken token2) + { + if (((object)token1) == null || ((object)token2) == null) + { + return object.Equals(token1, token2); + } + + return token1.Equals(token2); + } + + public static bool operator !=( + OptimisticConcurrencyControlToken token1, + OptimisticConcurrencyControlToken token2) + { + if (((object)token1) == null || ((object)token2) == null) + { + return !object.Equals(token1, token2); + } + + return !token1.Equals(token2); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/Storage.cs b/src/Catalog/Persistence/Storage.cs new file mode 100644 index 000000000..ba8197ed0 --- /dev/null +++ b/src/Catalog/Persistence/Storage.cs @@ -0,0 +1,241 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Newtonsoft.Json; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public abstract class Storage : IStorage + { + public Uri BaseAddress { get; protected set; } + public bool Verbose { get; protected set; } + + public Storage(Uri baseAddress) + { + string s = baseAddress.OriginalString.TrimEnd('/') + '/'; + BaseAddress = new Uri(s); + } + + public override string ToString() + { + return BaseAddress.ToString(); + } + + protected abstract Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken); + protected abstract Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken); + protected abstract Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken); + protected abstract Task OnDeleteAsync(Uri resourceUri, DeleteRequestOptions deleteRequestOptions, CancellationToken cancellationToken); + + public virtual Task GetOptimisticConcurrencyControlTokenAsync( + Uri resourceUri, + CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + public async Task CopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + TraceMethod(nameof(CopyAsync), sourceUri); + + var stopwatch = Stopwatch.StartNew(); + + try + { + await OnCopyAsync(sourceUri, destinationStorage, destinationUri, destinationProperties, cancellationToken); + } + catch (Exception e) + { + TraceException(nameof(CopyAsync), sourceUri, e); + throw; + } + + TraceExecutionTime(nameof(CopyAsync), sourceUri, stopwatch.ElapsedMilliseconds); + } + + public async Task SaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + { + TraceMethod(nameof(SaveAsync), resourceUri); + + var stopwatch = Stopwatch.StartNew(); + + try + { + await OnSaveAsync(resourceUri, content, cancellationToken); + } + catch (Exception e) + { + TraceException(nameof(SaveAsync), resourceUri, e); + throw; + } + + TraceExecutionTime(nameof(SaveAsync), resourceUri, stopwatch.ElapsedMilliseconds); + } + + public async Task LoadAsync(Uri resourceUri, CancellationToken cancellationToken) + { + StorageContent storageContent = null; + + TraceMethod(nameof(LoadAsync), resourceUri); + + var stopwatch = Stopwatch.StartNew(); + + try + { + storageContent = await OnLoadAsync(resourceUri, cancellationToken); + } + catch (Exception e) + { + TraceException(nameof(LoadAsync), resourceUri, e); + throw; + } + + TraceExecutionTime(nameof(LoadAsync), resourceUri, stopwatch.ElapsedMilliseconds); + + return storageContent; + } + + public async Task DeleteAsync(Uri resourceUri, CancellationToken cancellationToken, DeleteRequestOptions deleteRequestOptions = null) + { + TraceMethod(nameof(DeleteAsync), resourceUri); + + var stopwatch = Stopwatch.StartNew(); + + try + { + await OnDeleteAsync(resourceUri, deleteRequestOptions, cancellationToken); + } + catch (StorageException e) + { + WebException webException = e.InnerException as WebException; + if (webException != null) + { + HttpStatusCode statusCode = ((HttpWebResponse)webException.Response).StatusCode; + if (statusCode != HttpStatusCode.NotFound) + { + throw; + } + } + else + { + throw; + } + } + catch (Exception e) + { + TraceException(nameof(DeleteAsync), resourceUri, e); + throw; + } + + TraceExecutionTime(nameof(DeleteAsync), resourceUri, stopwatch.ElapsedMilliseconds); + } + + public async Task LoadStringAsync(Uri resourceUri, CancellationToken cancellationToken) + { + StorageContent content = await LoadAsync(resourceUri, cancellationToken); + if (content == null) + { + return null; + } + else + { + using (Stream stream = content.GetContentStream()) + { + StreamReader reader = new StreamReader(stream); + return await reader.ReadToEndAsync(); + } + } + } + + public abstract Task> ListAsync(CancellationToken cancellationToken); + + public abstract bool Exists(string fileName); + + public Uri ResolveUri(string relativeUri) + { + return new Uri(BaseAddress, relativeUri); + } + + /// + /// It will return false if there are changes between the source and destination. + /// + /// The first uri. + /// The second uri. + /// Default returns false. + public virtual Task AreSynchronized(Uri firstResourceUri, Uri secondResourceUri) + { + return Task.FromResult(false); + } + + protected string GetName(Uri uri) + { + var address = Uri.UnescapeDataString(BaseAddress.GetLeftPart(UriPartial.Path)); + if (!address.EndsWith("/")) + { + address += "/"; + } + var uriString = uri.ToString(); + + int baseAddressLength = address.Length; + + var name = uriString.Substring(baseAddressLength); + if (name.Contains("#")) + { + name = name.Substring(0, name.IndexOf("#")); + } + return name; + } + + public virtual Uri GetUri(string name) + { + string address = BaseAddress.ToString(); + if (!address.EndsWith("/")) + { + address += "/"; + } + address += name.Replace("\\", "/").TrimStart('/'); + + return new Uri(address); + } + + protected void TraceMethod(string method, Uri resourceUri) + { + if (Verbose) + { + //The Uri depends on the storage implementation. + Uri storageUri = GetUri(GetName(resourceUri)); + Trace.WriteLine(String.Format("{0} {1}", method, storageUri)); + } + } + + private string TraceException(string method, Uri resourceUri, Exception exception) + { + string message = $"{method} EXCEPTION: {GetUri(GetName(resourceUri))} {exception.ToString()}"; + Trace.WriteLine(message); + return message; + } + + private void TraceExecutionTime(string method, Uri resourceUri, long executionTimeInMilliseconds) + { + string message = JsonConvert.SerializeObject(new { MethodName = method, StreamUri = GetUri(GetName(resourceUri)), ExecutionTimeInMilliseconds = executionTimeInMilliseconds }); + Trace.WriteLine(message); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/StorageConstants.cs b/src/Catalog/Persistence/StorageConstants.cs new file mode 100644 index 000000000..4374b678a --- /dev/null +++ b/src/Catalog/Persistence/StorageConstants.cs @@ -0,0 +1,11 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public static class StorageConstants + { + public const string CacheControl = "CacheControl"; + public const string ContentType = "ContentType"; + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/StorageContent.cs b/src/Catalog/Persistence/StorageContent.cs new file mode 100644 index 000000000..665d7abc0 --- /dev/null +++ b/src/Catalog/Persistence/StorageContent.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System.IO; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public abstract class StorageContent + { + public string ContentType + { + get; + set; + } + + public string CacheControl + { + get; + set; + } + + public abstract Stream GetContentStream(); + } +} diff --git a/src/Catalog/Persistence/StorageFactory.cs b/src/Catalog/Persistence/StorageFactory.cs new file mode 100644 index 000000000..5da6be631 --- /dev/null +++ b/src/Catalog/Persistence/StorageFactory.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Protocol; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public abstract class StorageFactory : IStorageFactory + { + public StorageFactory() : this(throttle: null) + { + } + + public StorageFactory(IThrottle throttle) + { + Throttle = throttle ?? NullThrottle.Instance; + } + + public abstract Storage Create(string name = null); + + public Uri BaseAddress { get; protected set; } + + // For telemetry only + public Uri DestinationAddress { get; protected set; } + + public bool Verbose { get; protected set; } + + public IThrottle Throttle { get; } + + public override string ToString() + { + return BaseAddress.ToString(); + } + } +} \ No newline at end of file diff --git a/src/Catalog/Persistence/StorageListItem.cs b/src/Catalog/Persistence/StorageListItem.cs new file mode 100644 index 000000000..cb35ea97a --- /dev/null +++ b/src/Catalog/Persistence/StorageListItem.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public class StorageListItem + { + public Uri Uri { get; set; } + + public DateTime? LastModifiedUtc { get; set; } + + public StorageListItem(Uri uri, DateTime? lastModifiedUtc) + { + Uri = uri; + LastModifiedUtc = lastModifiedUtc; + } + } +} diff --git a/src/Catalog/Persistence/StreamStorageContent.cs b/src/Catalog/Persistence/StreamStorageContent.cs new file mode 100644 index 000000000..de6d5111a --- /dev/null +++ b/src/Catalog/Persistence/StreamStorageContent.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System.IO; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public class StreamStorageContent : StorageContent + { + public StreamStorageContent(Stream content, string contentType = "", string cacheControl = "") + { + Content = content; + ContentType = contentType; + CacheControl = cacheControl; + } + + public Stream Content + { + get; + set; + } + + public override Stream GetContentStream() + { + return Content; + } + } +} diff --git a/src/Catalog/Persistence/StringStorageContent.cs b/src/Catalog/Persistence/StringStorageContent.cs new file mode 100644 index 000000000..f07cd96a0 --- /dev/null +++ b/src/Catalog/Persistence/StringStorageContent.cs @@ -0,0 +1,40 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.IO; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + public class StringStorageContent : StorageContent + { + public StringStorageContent(string content, string contentType = "", string cacheControl = "") + { + Content = content; + ContentType = contentType; + CacheControl = cacheControl; + } + + public string Content + { + get; + set; + } + + public override Stream GetContentStream() + { + if (Content == null) + { + return null; + } + else + { + Stream stream = new MemoryStream(); + StreamWriter writer = new StreamWriter(stream); + writer.Write(Content); + writer.Flush(); + stream.Seek(0, SeekOrigin.Begin); + return stream; + } + } + } +} diff --git a/src/Catalog/Persistence/StringStorageContentWithAccessCondition.cs b/src/Catalog/Persistence/StringStorageContentWithAccessCondition.cs new file mode 100644 index 000000000..1d682a56a --- /dev/null +++ b/src/Catalog/Persistence/StringStorageContentWithAccessCondition.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.WindowsAzure.Storage; + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + /// + /// Allows specifying an for use by in a operation. + /// + public class StringStorageContentWithAccessCondition : StringStorageContent + { + public StringStorageContentWithAccessCondition( + string content, + AccessCondition accessCondition, + string contentType = "", + string cacheControl = "") + : base(content, contentType, cacheControl) + { + AccessCondition = accessCondition; + } + + public AccessCondition AccessCondition { get; } + } +} diff --git a/src/Catalog/Persistence/StringStorageContentWithETag.cs b/src/Catalog/Persistence/StringStorageContentWithETag.cs new file mode 100644 index 000000000..1d5580740 --- /dev/null +++ b/src/Catalog/Persistence/StringStorageContentWithETag.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog.Persistence +{ + /// + /// Used by to expose the ETag associated with a operation. + /// + public class StringStorageContentWithETag : StringStorageContent + { + public StringStorageContentWithETag( + string content, + string eTag, + string contentType = "", + string cacheControl = "") + : base(content, contentType, cacheControl) + { + ETag = eTag; + } + + public string ETag { get; } + } +} diff --git a/src/Catalog/ProcessCommitItemBatchAsync.cs b/src/Catalog/ProcessCommitItemBatchAsync.cs new file mode 100644 index 000000000..71664dc0e --- /dev/null +++ b/src/Catalog/ProcessCommitItemBatchAsync.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; + +namespace NuGet.Services.Metadata.Catalog +{ + public delegate Task ProcessCommitItemBatchAsync( + CollectorHttpClient client, + JToken context, + string packageId, + CatalogCommitItemBatch batch, + CatalogCommitItemBatch lastBatch, + CancellationToken cancellationToken); +} \ No newline at end of file diff --git a/src/Catalog/Properties/AssemblyInfo.cs b/src/Catalog/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..3ed6916e5 --- /dev/null +++ b/src/Catalog/Properties/AssemblyInfo.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Services.Metadata.Catalog")] +[assembly: AssemblyDescription("Create, edit, or read the package metadata catalog.")] +[assembly: Guid("e79af4e1-f145-4ffb-8774-737af1a81861")] +[assembly: AssemblyCompany(".NET Foundation")] +[assembly: AssemblyProduct("NuGet Services")] +[assembly: AssemblyCopyright("\x00a9 .NET Foundation. All rights reserved.")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +#if SIGNED_BUILD +[assembly: InternalsVisibleTo("CatalogTests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] +[assembly: InternalsVisibleTo("NgTests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] +#else +[assembly: InternalsVisibleTo("CatalogTests")] +[assembly: InternalsVisibleTo("NgTests")] +#endif \ No newline at end of file diff --git a/src/Catalog/ReadCursor.cs b/src/Catalog/ReadCursor.cs new file mode 100644 index 000000000..7e878673b --- /dev/null +++ b/src/Catalog/ReadCursor.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog +{ + public abstract class ReadCursor + { + public virtual DateTime Value { get; set; } + + public abstract Task LoadAsync(CancellationToken cancellationToken); + + public override string ToString() + { + return Value.ToString("O"); + } + } +} \ No newline at end of file diff --git a/src/Catalog/ReadOnlyGraph.cs b/src/Catalog/ReadOnlyGraph.cs new file mode 100644 index 000000000..bb4380619 --- /dev/null +++ b/src/Catalog/ReadOnlyGraph.cs @@ -0,0 +1,69 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using VDS.RDF; + +namespace NuGet.Services.Metadata.Catalog +{ + /// + /// An immutable graph supporting multi-threaded read-only access. According to dotNetRDF documentation, the + /// class is thread-safe when used in a read-only manner. + /// + /// + public class ReadOnlyGraph : Graph + { + private readonly bool _isReadOnly; + + public ReadOnlyGraph(IGraph graph) + { + Merge(graph); + _isReadOnly = true; + } + + public override bool Assert(IEnumerable ts) + { + ThrowIfReadOnly(); + return base.Assert(ts); + } + + public override bool Assert(Triple t) + { + ThrowIfReadOnly(); + return base.Assert(t); + } + + public override bool Retract(IEnumerable ts) + { + ThrowIfReadOnly(); + return base.Retract(ts); + } + + public override bool Retract(Triple t) + { + ThrowIfReadOnly(); + return base.Retract(t); + } + + public override void Merge(IGraph g) + { + ThrowIfReadOnly(); + base.Merge(g); + } + + public override void Merge(IGraph g, bool keepOriginalGraphUri) + { + ThrowIfReadOnly(); + base.Merge(g, keepOriginalGraphUri); + } + + private void ThrowIfReadOnly() + { + if (_isReadOnly) + { + throw new NotSupportedException("This RDF graph cannot be modified."); + } + } + } +} diff --git a/src/Catalog/ReadWriteCursor.cs b/src/Catalog/ReadWriteCursor.cs new file mode 100644 index 000000000..303af1fa5 --- /dev/null +++ b/src/Catalog/ReadWriteCursor.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog +{ + public abstract class ReadWriteCursor : ReadCursor + { + public abstract Task SaveAsync(CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/Catalog/ResourceSaveOperation.cs b/src/Catalog/ResourceSaveOperation.cs new file mode 100644 index 000000000..820efdfdf --- /dev/null +++ b/src/Catalog/ResourceSaveOperation.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog +{ + public class ResourceSaveOperation + { + public Uri ResourceUri { get; set; } + public Task SaveTask { get; set; } + } +} \ No newline at end of file diff --git a/src/Catalog/RetryWithExponentialBackoff.cs b/src/Catalog/RetryWithExponentialBackoff.cs new file mode 100644 index 000000000..c7118101d --- /dev/null +++ b/src/Catalog/RetryWithExponentialBackoff.cs @@ -0,0 +1,107 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog +{ + // See https://docs.microsoft.com/en-us/dotnet/standard/microservices-architecture/implement-resilient-applications/implement-custom-http-call-retries-exponential-backoff + public sealed class RetryWithExponentialBackoff : IHttpRetryStrategy + { + private readonly ushort _maximumRetries; + private readonly TimeSpan _delay; + private readonly TimeSpan _maximumDelay; + + internal RetryWithExponentialBackoff() + { + _maximumRetries = 3; + _delay = TimeSpan.FromSeconds(1); + _maximumDelay = TimeSpan.FromSeconds(10); + } + + public async Task SendAsync(HttpClient client, Uri address, CancellationToken cancellationToken) + { + var backoff = new ExponentialBackoff(_maximumRetries, _delay, _maximumDelay); + + while (true) + { + HttpResponseMessage httpResponse = null; + + try + { + httpResponse = await client.GetAsync(address, cancellationToken); + + httpResponse.EnsureSuccessStatusCode(); + + return httpResponse; + } + catch (Exception e) + { + httpResponse?.Dispose(); + if (IsTransientError(e, httpResponse)) + { + await backoff.Delay(); + } + else + { + throw; + } + } + } + } + + public static bool IsTransientError(Exception e, HttpResponseMessage response) + { + if (!(e is HttpRequestException || e is OperationCanceledException)) + { + return false; + } + + return response == null + || ((int)response.StatusCode >= 500 && + response.StatusCode != HttpStatusCode.NotImplemented && + response.StatusCode != HttpStatusCode.HttpVersionNotSupported); + } + + private sealed class ExponentialBackoff + { + private readonly ushort _maximumRetries; + private readonly TimeSpan _delay; + private readonly TimeSpan _maximumDelay; + private ushort _retries; + private int _power; + + internal ExponentialBackoff(ushort maximumRetries, TimeSpan delay, TimeSpan maximumDelay) + { + _maximumRetries = maximumRetries; + _delay = delay; + _maximumDelay = maximumDelay; + _retries = 0; + _power = 1; + } + + internal Task Delay() + { + if (_retries == _maximumRetries) + { + throw new TimeoutException("Maximum retry attempts exhausted."); + } + + ++_retries; + + if (_retries < 31) + { + _power = _power << 1; + } + + int delay = (int)Math.Min(_delay.TotalMilliseconds * (_power - 1) / 2, _maximumDelay.TotalMilliseconds); + + return Task.Delay(delay); + } + } + } +} \ No newline at end of file diff --git a/src/Catalog/Schema.cs b/src/Catalog/Schema.cs new file mode 100644 index 000000000..18ac0662e --- /dev/null +++ b/src/Catalog/Schema.cs @@ -0,0 +1,83 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog +{ + public static class Schema + { + public static class Prefixes + { + public static readonly string NuGet = "http://schema.nuget.org/schema#"; + public static readonly string Catalog = "http://schema.nuget.org/catalog#"; + public static readonly string Xsd = "http://www.w3.org/2001/XMLSchema#"; + public static readonly string Rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"; + } + + public static class DataTypes + { + public static readonly Uri PackageDetails = new Uri(Prefixes.NuGet + "PackageDetails"); + public static readonly Uri PackageDelete = new Uri(Prefixes.NuGet + "PackageDelete"); + + public static readonly Uri PackageEntry = new Uri(Prefixes.NuGet + "PackageEntry"); + + public static readonly Uri CatalogRoot = new Uri(Prefixes.Catalog + "CatalogRoot"); + public static readonly Uri CatalogPage = new Uri(Prefixes.Catalog + "CatalogPage"); + public static readonly Uri Permalink = new Uri(Prefixes.Catalog + "Permalink"); + public static readonly Uri AppendOnlyCatalog = new Uri(Prefixes.Catalog + "AppendOnlyCatalog"); + + public static readonly Uri Integer = new Uri(Prefixes.Xsd + "integer"); + public static readonly Uri DateTime = new Uri(Prefixes.Xsd + "dateTime"); + public static readonly Uri Boolean = new Uri(Prefixes.Xsd + "boolean"); + + public static readonly Uri Vulnerability = new Uri(Prefixes.NuGet + "Vulnerability"); + } + + public static class Predicates + { + public static readonly Uri Type = new Uri(Prefixes.Rdf + "type"); + + public static readonly Uri CatalogCommitId = new Uri(Prefixes.Catalog + "commitId"); + public static readonly Uri CatalogTimeStamp = new Uri(Prefixes.Catalog + "commitTimeStamp"); + public static readonly Uri CatalogItem = new Uri(Prefixes.Catalog + "item"); + public static readonly Uri CatalogCount = new Uri(Prefixes.Catalog + "count"); + public static readonly Uri CatalogParent = new Uri(Prefixes.Catalog + "parent"); + + public static readonly Uri Id = new Uri(Prefixes.NuGet + "id"); + public static readonly Uri Version = new Uri(Prefixes.NuGet + "version"); + public static readonly Uri OriginalId = new Uri(Prefixes.NuGet + "originalId"); + + public static readonly Uri PackageEntry = new Uri(Prefixes.NuGet + "packageEntry"); + public static readonly Uri FullName = new Uri(Prefixes.NuGet + "fullName"); + public static readonly Uri Name = new Uri(Prefixes.NuGet + "name"); + public static readonly Uri Length = new Uri(Prefixes.NuGet + "length"); + public static readonly Uri CompressedLength = new Uri(Prefixes.NuGet + "compressedLength"); + + // General-purpose fields used in C# explicitly (not just in the .nuspec to RDF XSLT) + public static readonly Uri Created = new Uri(Prefixes.NuGet + "created"); + + public static readonly Uri LastCreated = new Uri(Prefixes.NuGet + "lastCreated"); + public static readonly Uri LastEdited = new Uri(Prefixes.NuGet + "lastEdited"); + public static readonly Uri LastDeleted = new Uri(Prefixes.NuGet + "lastDeleted"); + public static readonly Uri Listed = new Uri(Prefixes.NuGet + "listed"); + + public static readonly Uri Published = new Uri(Prefixes.NuGet + "published"); + public static readonly Uri PackageHash = new Uri(Prefixes.NuGet + "packageHash"); + public static readonly Uri PackageHashAlgorithm = new Uri(Prefixes.NuGet + "packageHashAlgorithm"); + public static readonly Uri PackageSize = new Uri(Prefixes.NuGet + "packageSize"); + + public static readonly Uri Range = new Uri(Prefixes.NuGet + "range"); + + public static readonly Uri Deprecation = new Uri(Prefixes.NuGet + "deprecation"); + + public static readonly Uri Reasons = new Uri(Prefixes.NuGet + "reasons"); + public static readonly Uri Message = new Uri(Prefixes.NuGet + "message"); + public static readonly Uri AlternatePackage = new Uri(Prefixes.NuGet + "alternatePackage"); + + public static readonly Uri Vulnerability = new Uri(Prefixes.NuGet + "vulnerability"); + public static readonly Uri AdvisoryUrl = new Uri(Prefixes.NuGet + "advisoryUrl"); + public static readonly Uri Severity = new Uri(Prefixes.NuGet + "severity"); + } + } +} diff --git a/src/Catalog/SortingCollector.cs b/src/Catalog/SortingCollector.cs new file mode 100644 index 000000000..bc06124cd --- /dev/null +++ b/src/Catalog/SortingCollector.cs @@ -0,0 +1,71 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; + +namespace NuGet.Services.Metadata.Catalog +{ + public abstract class SortingCollector : CommitCollector where T : IEquatable + { + public SortingCollector( + Uri index, + ITelemetryService telemetryService, + Func handlerFunc = null, + IHttpRetryStrategy httpRetryStrategy = null) + : base(index, telemetryService, handlerFunc, httpRetryStrategy: httpRetryStrategy) + { + } + + protected override async Task OnProcessBatchAsync( + CollectorHttpClient client, + IEnumerable items, + JToken context, + DateTime commitTimeStamp, + bool isLastBatch, + CancellationToken cancellationToken) + { + var sortedItems = new Dictionary>(); + + foreach (CatalogCommitItem item in items) + { + T key = GetKey(item); + + IList itemList; + if (!sortedItems.TryGetValue(key, out itemList)) + { + itemList = new List(); + sortedItems.Add(key, itemList); + } + + itemList.Add(item); + } + + IList tasks = new List(); + + foreach (KeyValuePair> sortedBatch in sortedItems) + { + Task task = ProcessSortedBatchAsync(client, sortedBatch, context, cancellationToken); + + tasks.Add(task); + } + + await Task.WhenAll(tasks.ToArray()); + + return true; + } + + protected abstract T GetKey(CatalogCommitItem item); + + protected abstract Task ProcessSortedBatchAsync( + CollectorHttpClient client, + KeyValuePair> sortedBatch, + JToken context, + CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/Catalog/SortingIdVersionCollector.cs b/src/Catalog/SortingIdVersionCollector.cs new file mode 100644 index 000000000..b21fd0354 --- /dev/null +++ b/src/Catalog/SortingIdVersionCollector.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog +{ + public abstract class SortingIdVersionCollector : SortingCollector + { + public SortingIdVersionCollector(Uri index, ITelemetryService telemetryService, Func handlerFunc = null) + : base(index, telemetryService, handlerFunc) + { + } + + protected override FeedPackageIdentity GetKey(CatalogCommitItem item) + { + return new FeedPackageIdentity(item.PackageIdentity); + } + } +} \ No newline at end of file diff --git a/src/Catalog/StringInterner.cs b/src/Catalog/StringInterner.cs new file mode 100644 index 000000000..ff8defaef --- /dev/null +++ b/src/Catalog/StringInterner.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading; + +namespace NuGet.Services.Metadata.Catalog +{ + public class StringInterner + { + private readonly IDictionary _instances = new Dictionary(); + private readonly ReaderWriterLockSlim _lock = new ReaderWriterLockSlim(); + + public string Intern(string value) + { + var output = value; + + _lock.EnterUpgradeableReadLock(); + try + { + if (_instances.TryGetValue(value, out output)) + { + return output; + } + + _lock.EnterWriteLock(); + try + { + _instances.Add(value, value); + return value; + } + finally + { + _lock.ExitWriteLock(); + } + } + finally + { + _lock.ExitUpgradeableReadLock(); + } + } + } +} diff --git a/src/Catalog/Strings.Designer.cs b/src/Catalog/Strings.Designer.cs new file mode 100644 index 000000000..0a51f5ae6 --- /dev/null +++ b/src/Catalog/Strings.Designer.cs @@ -0,0 +1,144 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace NuGet.Services.Metadata.Catalog { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "15.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + internal class Strings { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Strings() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("NuGet.Services.Metadata.Catalog.Strings", typeof(Strings).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to The argument must be an instance of type {0}.. + /// + internal static string ArgumentMustBeInstanceOfType { + get { + return ResourceManager.GetString("ArgumentMustBeInstanceOfType", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The argument must not be null.. + /// + internal static string ArgumentMustNotBeNull { + get { + return ResourceManager.GetString("ArgumentMustNotBeNull", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The argument must not be null, empty, or whitespace.. + /// + internal static string ArgumentMustNotBeNullEmptyOrWhitespace { + get { + return ResourceManager.GetString("ArgumentMustNotBeNullEmptyOrWhitespace", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The argument must not be null or empty.. + /// + internal static string ArgumentMustNotBeNullOrEmpty { + get { + return ResourceManager.GetString("ArgumentMustNotBeNullOrEmpty", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The argument must be within the range from {0} (inclusive) to {1} (inclusive).. + /// + internal static string ArgumentOutOfRange { + get { + return ResourceManager.GetString("ArgumentOutOfRange", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to A failure occurred while processing a catalog batch.. + /// + internal static string BatchProcessingFailure { + get { + return ResourceManager.GetString("BatchProcessingFailure", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Multiple commits exist with the same commit timestamp but different commit ID's: {0}.. + /// + internal static string MultipleCommitIdsForSameCommitTimeStamp { + get { + return ResourceManager.GetString("MultipleCommitIdsForSameCommitTimeStamp", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The value of property '{0}' must be non-null and non-empty.. + /// + internal static string NonEmptyPropertyValueRequired { + get { + return ResourceManager.GetString("NonEmptyPropertyValueRequired", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The property '{0}' is required and is value must not be null.. + /// + internal static string PropertyRequired { + get { + return ResourceManager.GetString("PropertyRequired", resourceCulture); + } + } + } +} diff --git a/src/Catalog/Strings.resx b/src/Catalog/Strings.resx new file mode 100644 index 000000000..08c278616 --- /dev/null +++ b/src/Catalog/Strings.resx @@ -0,0 +1,147 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + The argument must be an instance of type {0}. + + + The argument must not be null. + + + The argument must not be null, empty, or whitespace. + + + The argument must not be null or empty. + + + The argument must be within the range from {0} (inclusive) to {1} (inclusive). + + + A failure occurred while processing a catalog batch. + + + Multiple commits exist with the same commit timestamp but different commit ID's: {0}. + + + The value of property '{0}' must be non-null and non-empty. + + + The property '{0}' is required and is value must not be null. + + \ No newline at end of file diff --git a/src/Catalog/Telemetry/ITelemetryService.cs b/src/Catalog/Telemetry/ITelemetryService.cs new file mode 100644 index 000000000..2f0995e11 --- /dev/null +++ b/src/Catalog/Telemetry/ITelemetryService.cs @@ -0,0 +1,44 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Versioning; + +namespace NuGet.Services.Metadata.Catalog +{ + public interface ITelemetryService + { + /// + /// Allows setting dimensions that will be added to all telemetry emited by the job. + /// + IDictionary GlobalDimensions { get; } + + void TrackCatalogIndexWriteDuration(TimeSpan duration, Uri uri); + void TrackCatalogIndexReadDuration(TimeSpan duration, Uri uri); + + IDisposable TrackIndexCommitDuration(); + void TrackIndexCommitTimeout(); + + void TrackHandlerFailedToProcessPackage(IPackagesContainerHandler handler, string packageId, NuGetVersion packageVersion); + void TrackPackageMissingHash(string packageId, NuGetVersion packageVersion); + void TrackPackageHasIncorrectHash(string packageId, NuGetVersion packageVersion); + void TrackPackageAlreadyHasHash(string packageId, NuGetVersion packageVersion); + void TrackPackageHashFixed(string packageId, NuGetVersion packageVersion); + + void TrackMetric(string name, ulong metric, IDictionary properties = null); + DurationMetric TrackDuration(string name, IDictionary properties = null); + IDisposable TrackExternalIconProcessingDuration(string packageId, string normalizedPackageVersion); + IDisposable TrackEmbeddedIconProcessingDuration(string packageId, string normalizedPackageVersion); + void TrackIconDeletionSuccess(string packageId, string normalizedPackageVersion); + void TrackIconDeletionFailure(string packageId, string normalizedPackageVersion); + void TrackExternalIconIngestionSuccess(string packageId, string normalizedPackageVersion); + void TrackExternalIconIngestionFailure(string packageId, string normalizedPackageVersion); + void TrackIconExtractionSuccess(string packageId, string normalizedPackageVersion); + void TrackIconExtractionFailure(string packageId, string normalizedPackageVersion); + IDisposable TrackGetPackageDetailsQueryDuration(Db2CatalogCursor cursor); + IDisposable TrackGetPackageQueryDuration(string packageId, string packageVersion); + } +} \ No newline at end of file diff --git a/src/Catalog/Telemetry/TelemetryConstants.cs b/src/Catalog/Telemetry/TelemetryConstants.cs new file mode 100644 index 000000000..38055f511 --- /dev/null +++ b/src/Catalog/Telemetry/TelemetryConstants.cs @@ -0,0 +1,61 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog +{ + public static class TelemetryConstants + { + public const string BatchItemCount = "BatchItemCount"; + public const string BlobModified = "BlobModified"; + public const string CatalogIndexReadDurationSeconds = "CatalogIndexReadDurationSeconds"; + public const string CatalogIndexWriteDurationSeconds = "CatalogIndexWriteDurationSeconds"; + public const string IndexCommitDurationSeconds = "IndexCommitDurationSeconds"; + public const string IndexCommitTimeout = "IndexCommitTimeout"; + public const string HandlerFailedToProcessPackage = "HandlerFailedToProcessPackage"; + public const string PackageMissingHash = "PackageMissingHash"; + public const string PackageHasIncorrectHash = "PackageHasIncorrectHash"; + public const string PackageAlreadyHasHash = "PackageAlreadyHasHash"; + public const string PackageHashFixed = "PackageHashFixed"; + public const string ContentBaseAddress = "ContentBaseAddress"; + public const string GalleryBaseAddress = "GalleryBaseAddress"; + public const string ContentLength = "ContentLength"; + public const string CreatedPackagesCount = "CreatedPackagesCount"; + public const string CreatedPackagesSeconds = "CreatedPackagesSeconds"; + public const string DeletedPackagesCount = "DeletedPackagesCount"; + public const string DeletedPackagesSeconds = "DeletedPackagesSeconds"; + public const string Destination = "Destination"; + public const string EditedPackagesCount = "EditedPackagesCount"; + public const string EditedPackagesSeconds = "EditedPackagesSeconds"; + public const string HttpHeaderDurationSeconds = "HttpHeaderDurationSeconds"; + public const string Id = "Id"; + public const string JobLoopSeconds = "JobLoopSeconds"; + public const string Method = "Method"; + public const string NonExistentBlob = "NonExistentBlob"; + public const string NonExistentPackageHash = "NonExistentPackageHash"; + public const string PackageBlobReadSeconds = "PackageBlobReadSeconds"; + public const string PackageDownloadSeconds = "PackageDownloadSeconds"; + public const string ProcessBatchSeconds = "ProcessBatchSeconds"; + public const string ProcessGraphsSeconds = "ProcessGraphsSeconds"; + public const string ProcessPackageDeleteSeconds = "ProcessPackageDeleteSeconds"; + public const string ProcessPackageDetailsSeconds = "ProcessPackageDetailsSeconds"; + public const string ProcessPackageVersionIndexSeconds = "ProcessPackageVersionIndexSeconds"; + public const string SizeInBytes = "SizeInBytes"; + public const string StatusCode = "StatusCode"; + public const string Success = "Success"; + public const string Uri = "Uri"; + public const string UsePackageSourceFallback = "UsePackageSourceFallback"; + public const string Version = "Version"; + public const string Handler = "Handler"; + public const string ExternalIconProcessing = "ExternalIconProcessing"; + public const string EmbeddedIconProcessing = "EmbeddedIconProcessing"; + public const string IconDeletionFailed = "IconDeletionFailed"; + public const string IconDeletionSucceeded = "IconDeletionSucceeded"; + public const string ExternalIconIngestionSucceeded = "ExternalIconIngestionSucceeded"; + public const string ExternalIconIngestionFailed = "ExternalIconIngestionFailed"; + public const string IconExtractionSucceeded = "IconExtractionSucceeded"; + public const string IconExtractionFailed = "IconExtractionFailed"; + public const string GetPackageDetailsSeconds = "GetPackageDetailsSeconds"; + public const string GetPackageSeconds = "GetPackageSeconds"; + public const string CursorValue = "CursorValue"; + } +} \ No newline at end of file diff --git a/src/Catalog/Telemetry/TelemetryHandler.cs b/src/Catalog/Telemetry/TelemetryHandler.cs new file mode 100644 index 000000000..180edd521 --- /dev/null +++ b/src/Catalog/Telemetry/TelemetryHandler.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog +{ + public class TelemetryHandler : DelegatingHandler + { + private readonly ITelemetryService _telemetryService; + + public TelemetryHandler(ITelemetryService telemetryService, HttpMessageHandler innerHandler) : base(innerHandler) + { + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var properties = new Dictionary() + { + { TelemetryConstants.Method, request.Method.ToString() }, + { TelemetryConstants.Uri, request.RequestUri.AbsoluteUri } + }; + + using (_telemetryService.TrackDuration(TelemetryConstants.HttpHeaderDurationSeconds, properties)) + { + var response = await base.SendAsync(request, cancellationToken); + + var contentLength = response.Content?.Headers?.ContentLength; + + properties[TelemetryConstants.StatusCode] = ((int)response.StatusCode).ToString(); + properties[TelemetryConstants.Success] = response.IsSuccessStatusCode.ToString(); + properties[TelemetryConstants.ContentLength] = contentLength == null ? "0" : contentLength.ToString(); + + return response; + } + } + } +} diff --git a/src/Catalog/Telemetry/TelemetryService.cs b/src/Catalog/Telemetry/TelemetryService.cs new file mode 100644 index 000000000..886ff0fda --- /dev/null +++ b/src/Catalog/Telemetry/TelemetryService.cs @@ -0,0 +1,305 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Versioning; + +namespace NuGet.Services.Metadata.Catalog +{ + public class TelemetryService : ITelemetryService + { + private readonly ITelemetryClient _telemetryClient; + + public IDictionary GlobalDimensions { get; } + + public TelemetryService(ITelemetryClient telemetryClient, IDictionary globalDimensions) + { + _telemetryClient = telemetryClient ?? throw new ArgumentNullException(nameof(telemetryClient)); + GlobalDimensions = globalDimensions ?? throw new ArgumentNullException(nameof(globalDimensions)); + } + + public void TrackCatalogIndexReadDuration(TimeSpan duration, Uri uri) + { + if (uri == null) + { + throw new ArgumentNullException(nameof(uri)); + } + + _telemetryClient.TrackMetric( + TelemetryConstants.CatalogIndexReadDurationSeconds, + duration.TotalSeconds, + new Dictionary + { + { TelemetryConstants.Uri, uri.AbsoluteUri }, + }); + } + + public void TrackCatalogIndexWriteDuration(TimeSpan duration, Uri uri) + { + if (uri == null) + { + throw new ArgumentNullException(nameof(uri)); + } + + _telemetryClient.TrackMetric( + TelemetryConstants.CatalogIndexWriteDurationSeconds, + duration.TotalSeconds, + new Dictionary + { + { TelemetryConstants.Uri, uri.AbsoluteUri }, + }); + } + + public IDisposable TrackIndexCommitDuration() + { + return _telemetryClient.TrackDuration(TelemetryConstants.IndexCommitDurationSeconds); + } + + public void TrackIndexCommitTimeout() + { + _telemetryClient.TrackMetric(TelemetryConstants.IndexCommitTimeout, 1); + } + + public void TrackHandlerFailedToProcessPackage(IPackagesContainerHandler handler, string packageId, NuGetVersion packageVersion) + { + if (handler == null) + { + throw new ArgumentNullException(nameof(handler)); + } + + if (string.IsNullOrEmpty(packageId)) + { + throw new ArgumentException($"The package id parameter is required", nameof(packageId)); + } + + if (packageVersion == null) + { + throw new ArgumentNullException(nameof(packageVersion)); + } + + _telemetryClient.TrackMetric( + TelemetryConstants.HandlerFailedToProcessPackage, + 1, + new Dictionary + { + { TelemetryConstants.Handler, handler.GetType().Name }, + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, packageVersion.ToNormalizedString() } + }); + } + + public void TrackPackageMissingHash(string packageId, NuGetVersion packageVersion) + { + if (string.IsNullOrEmpty(packageId)) + { + throw new ArgumentException($"The package id parameter is required", nameof(packageId)); + } + + if (packageVersion == null) + { + throw new ArgumentNullException(nameof(packageVersion)); + } + + _telemetryClient.TrackMetric( + TelemetryConstants.PackageMissingHash, + 1, + new Dictionary + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, packageVersion.ToNormalizedString() } + }); + } + + public void TrackPackageHasIncorrectHash(string packageId, NuGetVersion packageVersion) + { + if (string.IsNullOrEmpty(packageId)) + { + throw new ArgumentException($"The package id parameter is required", nameof(packageId)); + } + + if (packageVersion == null) + { + throw new ArgumentNullException(nameof(packageVersion)); + } + + _telemetryClient.TrackMetric( + TelemetryConstants.PackageHasIncorrectHash, + 1, + new Dictionary + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, packageVersion.ToNormalizedString() } + }); + } + + public void TrackPackageAlreadyHasHash(string packageId, NuGetVersion packageVersion) + { + if (string.IsNullOrEmpty(packageId)) + { + throw new ArgumentException($"The package id parameter is required", nameof(packageId)); + } + + if (packageVersion == null) + { + throw new ArgumentNullException(nameof(packageVersion)); + } + + _telemetryClient.TrackMetric( + TelemetryConstants.PackageAlreadyHasHash, + 1, + new Dictionary + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, packageVersion.ToNormalizedString() } + }); + } + + public void TrackPackageHashFixed(string packageId, NuGetVersion packageVersion) + { + if (string.IsNullOrEmpty(packageId)) + { + throw new ArgumentException($"The package id parameter is required", nameof(packageId)); + } + + if (packageVersion == null) + { + throw new ArgumentNullException(nameof(packageVersion)); + } + + _telemetryClient.TrackMetric( + TelemetryConstants.PackageHashFixed, + 1, + new Dictionary + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, packageVersion.ToNormalizedString() } + }); + } + + public void TrackMetric(string name, ulong metric, IDictionary properties = null) + { + _telemetryClient.TrackMetric(name, metric, properties); + } + + public virtual DurationMetric TrackDuration(string name, IDictionary properties = null) + { + return new DurationMetric(_telemetryClient, name, properties); + } + + public IDisposable TrackExternalIconProcessingDuration(string packageId, string normalizedPackageVersion) + { + return TrackDuration(TelemetryConstants.ExternalIconProcessing, new Dictionary + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, normalizedPackageVersion } + }); + } + + public IDisposable TrackEmbeddedIconProcessingDuration(string packageId, string normalizedPackageVersion) + { + return TrackDuration(TelemetryConstants.EmbeddedIconProcessing, new Dictionary + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, normalizedPackageVersion } + }); + } + + public void TrackIconDeletionSuccess(string packageId, string normalizedPackageVersion) + { + _telemetryClient.TrackMetric( + TelemetryConstants.IconDeletionSucceeded, + 1, + new Dictionary + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, normalizedPackageVersion } + }); + } + + public void TrackIconDeletionFailure(string packageId, string normalizedPackageVersion) + { + _telemetryClient.TrackMetric( + TelemetryConstants.IconDeletionFailed, + 1, + new Dictionary + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, normalizedPackageVersion } + }); + } + + public void TrackExternalIconIngestionFailure(string packageId, string normalizedPackageVersion) + { + _telemetryClient.TrackMetric( + TelemetryConstants.ExternalIconIngestionFailed, + 1, + new Dictionary + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, normalizedPackageVersion } + }); + } + + public void TrackExternalIconIngestionSuccess(string packageId, string normalizedPackageVersion) + { + _telemetryClient.TrackMetric( + TelemetryConstants.ExternalIconIngestionSucceeded, + 1, + new Dictionary + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, normalizedPackageVersion } + }); + } + + public void TrackIconExtractionSuccess(string packageId, string normalizedPackageVersion) + { + _telemetryClient.TrackMetric( + TelemetryConstants.IconExtractionSucceeded, + 1, + new Dictionary + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, normalizedPackageVersion } + }); + } + + public void TrackIconExtractionFailure(string packageId, string normalizedPackageVersion) + { + _telemetryClient.TrackMetric( + TelemetryConstants.IconExtractionFailed, + 1, + new Dictionary + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, normalizedPackageVersion } + }); + } + + public IDisposable TrackGetPackageDetailsQueryDuration(Db2CatalogCursor cursor) + { + var properties = new Dictionary() + { + { TelemetryConstants.Method, cursor.ColumnName }, + { TelemetryConstants.BatchItemCount, cursor.Top.ToString() }, + { TelemetryConstants.CursorValue, cursor.CursorValue.ToString("O") } + }; + + return _telemetryClient.TrackDuration(TelemetryConstants.GetPackageDetailsSeconds, properties); + } + + public IDisposable TrackGetPackageQueryDuration(string packageId, string packageVersion) + { + var properties = new Dictionary() + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, packageVersion } + }; + + return _telemetryClient.TrackDuration(TelemetryConstants.GetPackageSeconds, properties); + } + } +} \ No newline at end of file diff --git a/src/Catalog/TransientException.cs b/src/Catalog/TransientException.cs new file mode 100644 index 000000000..6390fa65d --- /dev/null +++ b/src/Catalog/TransientException.cs @@ -0,0 +1,47 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Runtime.Serialization; + +namespace NuGet.Services.Metadata.Catalog +{ + /// + /// Thrown for intermittent issues where the service can continue without the process being shutdown. + /// + [Serializable] + public class TransientException : System.Exception + { + public TransientException(string message ) : base(message) + { + } + + public TransientException(string message, Exception innerException) : base(message, innerException) + { + } + + protected TransientException(SerializationInfo info, StreamingContext context): base(info, context) + { + } + } + + /// + /// Thrown when an times out. + /// + [Serializable] + public class HttpClientTimeoutException : TransientException + { + public HttpClientTimeoutException(string message) : base(message) + { + } + + public HttpClientTimeoutException(string message, Exception innerException) : base(message, innerException) + { + } + + protected HttpClientTimeoutException(SerializationInfo info, StreamingContext context) : base(info, context) + { + } + } +} diff --git a/src/Catalog/Utilities.cs b/src/Catalog/Utilities.cs new file mode 100644 index 000000000..10e516b0e --- /dev/null +++ b/src/Catalog/Utilities.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog +{ + public static class Utilities + { + public static Uri GetNugetCacheBustingUri(Uri originalUri) + { + return GetNugetCacheBustingUri(originalUri, DateTime.UtcNow.ToString()); + } + + /// + /// Adding the timestamp to the URI as a query string ensures a cache miss with the CDN + /// + /// Returns a URI which ensures a cache miss with the CDN + public static Uri GetNugetCacheBustingUri(Uri originalUri, string timestamp) + { + var uriBuilder = new UriBuilder(originalUri); + uriBuilder.Query = "nuget-cache=" + timestamp; + return uriBuilder.Uri; + } + } +} diff --git a/src/Catalog/ValidatePackageHashHandler.cs b/src/Catalog/ValidatePackageHashHandler.cs new file mode 100644 index 000000000..85d651f6c --- /dev/null +++ b/src/Catalog/ValidatePackageHashHandler.cs @@ -0,0 +1,72 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services.Metadata.Catalog +{ + /// + /// Validates that packages in the packages container have the correct MD5 Content hash on their blob's properties. + /// + public class ValidatePackageHashHandler : IPackagesContainerHandler + { + private readonly HttpClient _httpClient; + private readonly ITelemetryService _telemetryService; + private readonly ILogger _logger; + + public ValidatePackageHashHandler( + HttpClient httpClient, + ITelemetryService telemetryService, + ILogger logger) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ProcessPackageAsync(CatalogIndexEntry packageEntry, ICloudBlockBlob blob) + { + await blob.FetchAttributesAsync(CancellationToken.None); + + if (blob.ContentMD5 == null) + { + _telemetryService.TrackPackageMissingHash(packageEntry.Id, packageEntry.Version); + + _logger.LogError( + "Package {PackageId} {PackageVersion} has a null Content MD5 hash!", + packageEntry.Id, + packageEntry.Version); + return; + } + + // Download the blob and calculate its hash. We use HttpClient to download blobs as Azure Blob Sotrage SDK + // occassionally hangs. See: https://github.com/Azure/azure-storage-net/issues/470 + string hash; + using (var hashAlgorithm = MD5.Create()) + using (var packageStream = await _httpClient.GetStreamAsync(blob.Uri)) + { + var hashBytes = hashAlgorithm.ComputeHash(packageStream); + + hash = Convert.ToBase64String(hashBytes); + } + + if (blob.ContentMD5 != hash) + { + _telemetryService.TrackPackageHasIncorrectHash(packageEntry.Id, packageEntry.Version); + + _logger.LogError( + "Package {PackageId} {PackageVersion} has an incorrect Content MD5 hash! Expected: '{ExpectedHash}', actual: '{ActualHash}'", + packageEntry.Id, + packageEntry.Version, + hash, + blob.ContentMD5); + } + } + } +} diff --git a/src/Catalog/VerboseHandler.cs b/src/Catalog/VerboseHandler.cs new file mode 100644 index 000000000..15ea9856e --- /dev/null +++ b/src/Catalog/VerboseHandler.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Diagnostics; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog +{ + public class VerboseHandler : DelegatingHandler + { + public VerboseHandler(HttpMessageHandler innerHandler) : base(innerHandler) + { + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Stopwatch sw = new Stopwatch(); + sw.Start(); + HttpResponseMessage response = await base.SendAsync(request, cancellationToken); + sw.Stop(); + Trace.TraceInformation("HTTP {0}_{1}_{2}", request.Method, request.RequestUri, sw.ElapsedMilliseconds); + return response; + } + } +} diff --git a/src/Catalog/context/Catalog.json b/src/Catalog/context/Catalog.json new file mode 100644 index 000000000..1b34044f6 --- /dev/null +++ b/src/Catalog/context/Catalog.json @@ -0,0 +1,15 @@ +{ + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "details": "catalog:details", + "catalog:commitTimeStamp": { "@type": "xsd:dateTime" }, + "published": { "@type": "xsd:dateTime" }, + "categories": { "@container" : "@set" }, + "entries": { "@container" : "@set" }, + "links": { "@container" : "@set" }, + "tags": { "@container": "@set" }, + "packageContent": { "@type": "@id" } + } +} diff --git a/src/Catalog/context/Container.json b/src/Catalog/context/Container.json new file mode 100644 index 000000000..bc5ebb4ed --- /dev/null +++ b/src/Catalog/context/Container.json @@ -0,0 +1,12 @@ +{ + "@context" : { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget" : "http://schema.nuget.org/schema#", + "items" : { "@id": "item", "@container" : "@set" }, + "parent" : { "@type" : "@id" }, + "commitTimeStamp" : { "@type" : "http://www.w3.org/2001/XMLSchema#dateTime" }, + "nuget:lastCreated" : { "@type" : "http://www.w3.org/2001/XMLSchema#dateTime" }, + "nuget:lastEdited" : { "@type" : "http://www.w3.org/2001/XMLSchema#dateTime" }, + "nuget:lastDeleted" : { "@type" : "http://www.w3.org/2001/XMLSchema#dateTime" } + } +} diff --git a/src/Catalog/context/EMA.json b/src/Catalog/context/EMA.json new file mode 100644 index 000000000..209ab7b77 --- /dev/null +++ b/src/Catalog/context/EMA.json @@ -0,0 +1,17 @@ +{ + "nuget" : "http://nuget.org/schema#", + "dcterms" : "http://purl.org/dc/terms/", + "siena" : "http://siena.com/schema#", + "id" : "nuget:id", + "version" : "nuget:version", + "abstract" : "dcterms:abstract", + "description" : "dcterms:description", + "title" : "dcterms:title", + "created" : "dcterms:created", + "creator" : "dcterms:creator", + "language" : { "@id" : "dcterms:language", "@container" : "@set" }, + "orientation" : "siena:orientation", + "aspectRatio" : "siena:aspectRatio", + "width" : "siena:width", + "height" : "siena:height" +} diff --git a/src/Catalog/context/PackageDetails.json b/src/Catalog/context/PackageDetails.json new file mode 100644 index 000000000..e4103a3c2 --- /dev/null +++ b/src/Catalog/context/PackageDetails.json @@ -0,0 +1,19 @@ +{ + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies" : { "@id" : "dependency", "@container" : "@set" }, + "dependencyGroups" : { "@id" : "dependencyGroup", "@container" : "@set" }, + "packageEntries": { "@id": "packageEntry", "@container": "@set" }, + "packageTypes": { "@id": "packageType", "@container": "@set" }, + "supportedFrameworks": { "@id": "supportedFramework", "@container": "@set" }, + "tags": { "@id": "tag", "@container": "@set" }, + "vulnerabilities": { "@id": "vulnerability", "@container": "@set" }, + "published": { "@type": "xsd:dateTime" }, + "created": { "@type": "xsd:dateTime" }, + "lastEdited" : { "@type" : "xsd:dateTime" }, + "catalog:commitTimeStamp" : { "@type" : "xsd:dateTime" }, + "reasons" : { "@container" : "@set" } + } +} \ No newline at end of file diff --git a/src/Catalog/xslt/normalizeNuspecNamespace.xslt b/src/Catalog/xslt/normalizeNuspecNamespace.xslt new file mode 100644 index 000000000..624c655e1 --- /dev/null +++ b/src/Catalog/xslt/normalizeNuspecNamespace.xslt @@ -0,0 +1,15 @@ + + + + + + + + + + + + + diff --git a/src/Catalog/xslt/nuspec.xslt b/src/Catalog/xslt/nuspec.xslt new file mode 100644 index 000000000..82ea7462a --- /dev/null +++ b/src/Catalog/xslt/nuspec.xslt @@ -0,0 +1,371 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj index 141c317c9..a1c5ce3f5 100644 --- a/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj +++ b/src/Gallery.CredentialExpiration/Gallery.CredentialExpiration.csproj @@ -92,7 +92,7 @@ 5.8.4 - 2.74.0 + 2.75.0 diff --git a/src/Ng/App.config b/src/Ng/App.config new file mode 100644 index 000000000..9b0eaf48f --- /dev/null +++ b/src/Ng/App.config @@ -0,0 +1,19 @@ + + + +
+ + + + + + + + + + + + + + + diff --git a/src/Ng/Arguments.cs b/src/Ng/Arguments.cs new file mode 100644 index 000000000..76546f7d4 --- /dev/null +++ b/src/Ng/Arguments.cs @@ -0,0 +1,182 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Metadata.Catalog.Monitoring; + +namespace Ng +{ + public static class Arguments + { + #region Shared + public const char Prefix = '-'; + public const char Quote = '"'; + + public const string Gallery = "gallery"; + public const string InstrumentationKey = "instrumentationkey"; + public const string HeartbeatIntervalSeconds = "HeartbeatIntervalSeconds"; + public const string Path = "path"; + public const string Source = "source"; + public const string Verbose = "verbose"; + public const string InstanceName = "instanceName"; + + public const int DefaultInterval = 3; // seconds + public const string Interval = "interval"; + + public const int DefaultReinitializeIntervalSec = 60 * 60; // 1 hour + public const string ReinitializeIntervalSec = "ReinitializeIntervalSec"; + + public const string AzureStorageType = "azure"; + public const string FileStorageType = "file"; + + public const string ConnectionString = "connectionString"; + public const string ContentBaseAddress = "contentBaseAddress"; + public const string GalleryBaseAddress = "galleryBaseAddress"; + public const string StorageAccountName = "storageAccountName"; + public const string StorageBaseAddress = "storageBaseAddress"; + public const string StorageContainer = "storageContainer"; + public const string StorageKeyValue = "storageKeyValue"; + public const string StoragePath = "storagePath"; + public const string StorageQueueName = "storageQueueName"; + public const string StorageType = "storageType"; + + public const string StorageSuffix = "storageSuffix"; + public const string StorageOperationMaxExecutionTimeInSeconds = "storageOperationMaxExecutionTimeInSeconds"; + public const string StorageServerTimeoutInSeconds = "storageServerTimeoutInSeconds"; + public const string HttpClientTimeoutInSeconds = "httpClientTimeoutInSeconds"; + + public const string StorageAccountNamePreferredPackageSourceStorage = "storageAccountNamePreferredPackageSourceStorage"; + public const string StorageKeyValuePreferredPackageSourceStorage = "storageKeyValuePreferredPackageSourceStorage"; + public const string StorageContainerPreferredPackageSourceStorage = "storageContainerPreferredPackageSourceStorage"; + + public const string PreferAlternatePackageSourceStorage = "preferAlternatePackageSourceStorage"; + + public const string StorageUseServerSideCopy = "storageUseServerSideCopy"; + + #endregion + + #region Catalog2PackageFixup + public const string Verify = "verify"; + #endregion + + #region Db2Catalog + public const string StartDate = "startDate"; + public const string PackageContentUrlFormat = "packageContentUrlFormat"; + public const string CursorSize = "cursorSize"; + + public const string StorageAccountNameAuditing = "storageAccountNameAuditing"; + public const string StorageContainerAuditing = "storageContainerAuditing"; + public const string StorageKeyValueAuditing = "storageKeyValueAuditing"; + public const string StoragePathAuditing = "storagePathAuditing"; + public const string StorageTypeAuditing = "storageTypeAuditing"; + public const string SqlCommandTimeoutInSeconds = "sqlCommandTimeoutInSeconds"; + + public const string SkipCreatedPackagesProcessing = "skipCreatedPackagesProcessing"; + #endregion + + #region Monitoring + /// + /// The url of the service index. + /// + public const string Index = "index"; + + /// + /// The url of the cursor for . + /// + public const string RegistrationCursorUri = "registrationCursorUri"; + + /// + /// The url of the cursor for . + /// + public const string FlatContainerCursorUri = "flatContainerCursorUri"; + + /// + /// The argument prefix for the cursor of a . There are multiple search endpoints + /// so a parameter matching this prefix represents the cursor for a single instance. The suffix (after the prefix) + /// is the instance identifier, e.g. "usnc-a". + /// + public const string SearchCursorUriPrefix = "searchCursorUri-"; + + /// + /// The argument prefix for the base URL of a . There should be the same number of + /// parameters passed as with the same + /// set of suffixes. + /// + public const string SearchBaseUriPrefix = "searchBaseUri-"; + + /// + /// The folder in which es are saved by the . + /// Defaults to . + /// + public const string PackageStatusFolder = "statusFolder"; + + /// + /// Default value of . + /// + public const string PackageStatusFolderDefault = "status"; + + /// + /// If the queue contains more messages than this, the job will not requeue any invalid packages. + /// + public const string MaxRequeueQueueSize = "maxRequeueQueueSize"; + + /// + /// If true, packages are expected to have at least a repository signature. + /// + public const string RequireRepositorySignature = "requireRepositorySignature"; + + /// + /// Continue to poll for messages until this amount of time is elapsed. + /// Then, refresh the job loop. + /// + public const string QueueLoopDurationHours = "queueLoopDurationHours"; + + /// + /// When the queue is empty or processing a message fails, wait this long before polling more. + /// + public const string QueueDelaySeconds = "queueDelaySeconds"; + + /// + /// The number of parallel workers for . + /// + public const string WorkerCount = "workerCount"; + #endregion + + #region KeyVault + public const string VaultName = "vaultName"; + public const string UseManagedIdentity = "useManagedIdentity"; + + public const string ClientId = "clientId"; + + public const string StoreName = "storeName"; + public const string StoreLocation = "storeLocation"; + + public const string CertificateThumbprint = "certificateThumbprint"; + public const string ValidateCertificate = "validateCertificate"; + + public const string RefreshIntervalSec = "refreshIntervalSec"; + #endregion + + #region Lightning + public const string CompressedStorageAccountName = "compressedStorageAccountName"; + public const string CompressedStorageBaseAddress = "compressedStorageBaseAddress"; + public const string CompressedStorageContainer = "compressedStorageContainer"; + public const string CompressedStorageKeyValue = "compressedStorageKeyValue"; + public const string CompressedStoragePath = "compressedStoragePath"; + + public const string SemVer2StorageAccountName = "semVer2StorageAccountName"; + public const string SemVer2StorageBaseAddress = "semVer2StorageBaseAddress"; + public const string SemVer2StorageContainer = "semVer2StorageContainer"; + public const string SemVer2StorageKeyValue = "semVer2StorageKeyValue"; + public const string SemVer2StoragePath = "semVer2StoragePath"; + + public const string FlatContainerName = "flatContainerName"; + + public const string Command = "command"; + public const string OutputFolder = "outputFolder"; + public const string TemplateFile = "templateFile"; + public const string BatchSize = "batchSize"; + public const string IndexFile = "indexFile"; + public const string CursorFile = "cursorFile"; + #endregion + } +} \ No newline at end of file diff --git a/src/Ng/Catalog2Dnx.nuspec b/src/Ng/Catalog2Dnx.nuspec new file mode 100644 index 000000000..34b17223c --- /dev/null +++ b/src/Ng/Catalog2Dnx.nuspec @@ -0,0 +1,16 @@ + + + + Catalog2Dnx + $version$ + .NET Foundation + .NET Foundation + The Catalog2Dnx job. + Copyright .NET Foundation + + + + + + + \ No newline at end of file diff --git a/src/Ng/Catalog2Monitoring.nuspec b/src/Ng/Catalog2Monitoring.nuspec new file mode 100644 index 000000000..28583be3f --- /dev/null +++ b/src/Ng/Catalog2Monitoring.nuspec @@ -0,0 +1,16 @@ + + + + Catalog2Monitoring + $version$ + .NET Foundation + .NET Foundation + The Catalog2Monitoring job. + Copyright .NET Foundation + + + + + + + \ No newline at end of file diff --git a/src/Ng/Catalog2icon.nuspec b/src/Ng/Catalog2icon.nuspec new file mode 100644 index 000000000..9a35691e0 --- /dev/null +++ b/src/Ng/Catalog2icon.nuspec @@ -0,0 +1,16 @@ + + + + Catalog2icon + $version$ + .NET Foundation + .NET Foundation + The Catalog2icon job. + Copyright .NET Foundation + + + + + + + \ No newline at end of file diff --git a/src/Ng/CommandHelpers.cs b/src/Ng/CommandHelpers.cs new file mode 100644 index 000000000..fdc0dd29e --- /dev/null +++ b/src/Ng/CommandHelpers.cs @@ -0,0 +1,365 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography.X509Certificates; +using Microsoft.Extensions.Logging; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Auth; +using NuGet.Protocol; +using NuGet.Services.Configuration; +using NuGet.Services.KeyVault; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Services.Storage; +using CatalogAzureStorage = NuGet.Services.Metadata.Catalog.Persistence.AzureStorage; +using CatalogAzureStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.AzureStorageFactory; +using CatalogFileStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.FileStorageFactory; +using CatalogStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.StorageFactory; +using ICatalogStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.IStorageFactory; + +namespace Ng +{ + public static class CommandHelpers + { + private static readonly int DefaultKeyVaultSecretCachingTimeout = 60 * 60 * 6; // 6 hours; + private static readonly HashSet NotInjectedKeys = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "connectionString", + }; + + public static IDictionary GetArguments(string[] args, int start, out ISecretInjector secretInjector) + { + var unprocessedArguments = new Dictionary(StringComparer.OrdinalIgnoreCase); + + if ((args.Length - 1) % 2 != 0) + { + Trace.TraceError("Unexpected number of arguments"); + secretInjector = null; + + return null; + } + + for (var i = start; i < args.Length; i += 2) + { + // Remove hyphen from the beginning of the argument name. + var argumentName = args[i].TrimStart(Arguments.Prefix); + // Remove quotes (if any) from the start and end of the argument value. + var argumentValue = args[i + 1].Trim(Arguments.Quote); + unprocessedArguments.Add(argumentName, argumentValue); + } + + secretInjector = GetSecretInjector(unprocessedArguments); + + return new SecretDictionary(secretInjector, unprocessedArguments, NotInjectedKeys); + } + + private static void TraceRequiredArgument(string name) + { + Trace.TraceError("Required argument \"{0}\" not provided", name); + } + + private static ISecretInjector GetSecretInjector(IDictionary arguments) + { + ISecretReader secretReader; + + var vaultName = arguments.GetOrDefault(Arguments.VaultName); + if (string.IsNullOrEmpty(vaultName)) + { + secretReader = new EmptySecretReader(); + } + else + { + var useManagedIdentity = arguments.GetOrDefault(Arguments.UseManagedIdentity); + KeyVaultConfiguration keyVaultConfig; + if (useManagedIdentity) + { + keyVaultConfig = new KeyVaultConfiguration(vaultName); + } + else + { + var clientId = arguments.GetOrThrow(Arguments.ClientId); + var certificateThumbprint = arguments.GetOrThrow(Arguments.CertificateThumbprint); + var storeName = arguments.GetOrDefault(Arguments.StoreName, StoreName.My); + var storeLocation = arguments.GetOrDefault(Arguments.StoreLocation, StoreLocation.LocalMachine); + var shouldValidateCert = arguments.GetOrDefault(Arguments.ValidateCertificate, true); + + var keyVaultCertificate = CertificateUtility.FindCertificateByThumbprint(storeName, storeLocation, certificateThumbprint, shouldValidateCert); + keyVaultConfig = new KeyVaultConfiguration(vaultName, clientId, keyVaultCertificate); + } + + secretReader = new CachingSecretReader(new KeyVaultReader(keyVaultConfig), + arguments.GetOrDefault(Arguments.RefreshIntervalSec, DefaultKeyVaultSecretCachingTimeout)); + } + + return new SecretInjector(secretReader); + } + + public static void AssertAzureStorage(IDictionary arguments) + { + if (arguments.GetOrThrow(Arguments.StorageType) != Arguments.AzureStorageType) + { + throw new ArgumentException("Only Azure storage is supported!"); + } + } + + public static CatalogStorageFactory CreateStorageFactory( + IDictionary arguments, + bool verbose, + IThrottle throttle = null) + { + IDictionary names = new Dictionary + { + { Arguments.StorageBaseAddress, Arguments.StorageBaseAddress }, + { Arguments.StorageAccountName, Arguments.StorageAccountName }, + { Arguments.StorageKeyValue, Arguments.StorageKeyValue }, + { Arguments.StorageContainer, Arguments.StorageContainer }, + { Arguments.StoragePath, Arguments.StoragePath }, + { Arguments.StorageSuffix, Arguments.StorageSuffix }, + { Arguments.StorageUseServerSideCopy, Arguments.StorageUseServerSideCopy }, + { Arguments.StorageOperationMaxExecutionTimeInSeconds, Arguments.StorageOperationMaxExecutionTimeInSeconds }, + { Arguments.StorageServerTimeoutInSeconds, Arguments.StorageServerTimeoutInSeconds } + }; + + return CreateStorageFactoryImpl( + arguments, + names, + verbose, + compressed: false, + throttle: throttle); + } + + public static CatalogStorageFactory CreateSuffixedStorageFactory( + string suffix, + IDictionary arguments, + bool verbose, + IThrottle throttle = null) + { + if (string.IsNullOrEmpty(suffix)) + { + throw new ArgumentNullException(nameof(suffix)); + } + + IDictionary names = new Dictionary + { + { Arguments.StorageBaseAddress, Arguments.StorageBaseAddress + suffix }, + { Arguments.StorageAccountName, Arguments.StorageAccountName + suffix }, + { Arguments.StorageKeyValue, Arguments.StorageKeyValue + suffix }, + { Arguments.StorageContainer, Arguments.StorageContainer + suffix }, + { Arguments.StoragePath, Arguments.StoragePath + suffix }, + { Arguments.StorageSuffix, Arguments.StorageSuffix + suffix }, + { Arguments.StorageUseServerSideCopy, Arguments.StorageUseServerSideCopy + suffix }, + { Arguments.StorageOperationMaxExecutionTimeInSeconds, Arguments.StorageOperationMaxExecutionTimeInSeconds + suffix }, + { Arguments.StorageServerTimeoutInSeconds, Arguments.StorageServerTimeoutInSeconds } + }; + + return CreateStorageFactoryImpl( + arguments, + names, + verbose, + compressed: false, + throttle: throttle); + } + + private static CatalogStorageFactory CreateStorageFactoryImpl( + IDictionary arguments, + IDictionary argumentNameMap, + bool verbose, + bool compressed, + IThrottle throttle = null) + { + Uri storageBaseAddress = null; + var storageBaseAddressStr = arguments.GetOrDefault(argumentNameMap[Arguments.StorageBaseAddress]); + if (!string.IsNullOrEmpty(storageBaseAddressStr)) + { + storageBaseAddressStr = storageBaseAddressStr.TrimEnd('/') + "/"; + + storageBaseAddress = new Uri(storageBaseAddressStr); + } + + var storageType = arguments.GetOrThrow(Arguments.StorageType); + + if (storageType.Equals(Arguments.FileStorageType, StringComparison.InvariantCultureIgnoreCase)) + { + var storagePath = arguments.GetOrThrow(argumentNameMap[Arguments.StoragePath]); + + if (storageBaseAddress != null) + { + return new CatalogFileStorageFactory(storageBaseAddress, storagePath, verbose); + } + + TraceRequiredArgument(argumentNameMap[Arguments.StorageBaseAddress]); + return null; + } + + if (Arguments.AzureStorageType.Equals(storageType, StringComparison.InvariantCultureIgnoreCase)) + { + var storageAccountName = arguments.GetOrThrow(argumentNameMap[Arguments.StorageAccountName]); + var storageKeyValue = arguments.GetOrThrow(argumentNameMap[Arguments.StorageKeyValue]); + var storageContainer = arguments.GetOrThrow(argumentNameMap[Arguments.StorageContainer]); + var storagePath = arguments.GetOrDefault(argumentNameMap[Arguments.StoragePath]); + var storageSuffix = arguments.GetOrDefault(argumentNameMap[Arguments.StorageSuffix]); + var storageOperationMaxExecutionTime = MaxExecutionTime(arguments.GetOrDefault(argumentNameMap[Arguments.StorageOperationMaxExecutionTimeInSeconds])); + var storageServerTimeout = MaxExecutionTime(arguments.GetOrDefault(argumentNameMap[Arguments.StorageServerTimeoutInSeconds])); + var storageUseServerSideCopy = arguments.GetOrDefault(argumentNameMap[Arguments.StorageUseServerSideCopy]); + + var credentials = new StorageCredentials(storageAccountName, storageKeyValue); + + var account = string.IsNullOrEmpty(storageSuffix) ? + new CloudStorageAccount(credentials, useHttps: true) : + new CloudStorageAccount(credentials, storageSuffix, useHttps: true); + + return new CatalogAzureStorageFactory( + account, + storageContainer, + storageOperationMaxExecutionTime, + storageServerTimeout, + storagePath, + storageBaseAddress, + storageUseServerSideCopy, + compressed, + verbose, + initializeContainer: true, + throttle: throttle ?? NullThrottle.Instance); + } + throw new ArgumentException($"Unrecognized storageType \"{storageType}\""); + } + + private static TimeSpan MaxExecutionTime(int seconds) + { + if (seconds < 0) + { + throw new ArgumentException($"{nameof(seconds)} cannot be negative."); + } + if (seconds == 0) + { + return CatalogAzureStorage.DefaultMaxExecutionTime; + } + return TimeSpan.FromSeconds(seconds); + } + + public static Func GetHttpMessageHandlerFactory( + ITelemetryService telemetryService, + bool verbose, + string catalogBaseAddress = null, + string storageBaseAddress = null) + { + Func defaultHandlerFunc = () => + { + var httpClientHandler = new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + }; + + return new TelemetryHandler(telemetryService, httpClientHandler); + }; + + Func handlerFunc = defaultHandlerFunc; + + if (verbose) + { + handlerFunc = + () => + catalogBaseAddress != null + ? new VerboseHandler(new StorageAccessHandler(catalogBaseAddress, storageBaseAddress, defaultHandlerFunc())) + : new VerboseHandler(defaultHandlerFunc()); + } + + return handlerFunc; + } + + public static EndpointConfiguration GetEndpointConfiguration(IDictionary arguments) + { + var registrationCursorUri = arguments.GetOrThrow(Arguments.RegistrationCursorUri); + var flatContainerCursorUri = arguments.GetOrThrow(Arguments.FlatContainerCursorUri); + + var instanceNameToSearchBaseUri = GetSuffixToUri(arguments, Arguments.SearchBaseUriPrefix); + var instanceNameToSearchCursorUri = GetSuffixToUri(arguments, Arguments.SearchCursorUriPrefix); + var instanceNameToSearchConfig = new Dictionary(); + foreach (var pair in instanceNameToSearchBaseUri) + { + var instanceName = pair.Key; + + // Find all cursors with an instance name starting with the search base URI instance name. We do this + // because there may be multiple potential cursors representing the state of a search service. + var matchingCursors = instanceNameToSearchCursorUri.Keys.Where(x => x.StartsWith(instanceName)).ToList(); + + if (!matchingCursors.Any()) + { + throw new ArgumentException( + $"The -{Arguments.SearchBaseUriPrefix}{instanceName} argument does not have any matching " + + $"-{Arguments.SearchCursorUriPrefix}{instanceName}* arguments."); + } + + instanceNameToSearchConfig[instanceName] = new SearchEndpointConfiguration( + matchingCursors.Select(x => instanceNameToSearchCursorUri[x]).ToList(), + pair.Value); + + foreach (var key in matchingCursors) + { + instanceNameToSearchCursorUri.Remove(key); + } + } + + // See if there are any search cursor URI arguments left over and error out. Better to fail than to ignore + // an argument that the user expected to be relevant. + if (instanceNameToSearchCursorUri.Any()) + { + throw new ArgumentException( + $"There are -{Arguments.SearchCursorUriPrefix}* arguments without matching " + + $"-{Arguments.SearchBaseUriPrefix}* arguments. The unmatched suffixes were: {string.Join(", ", instanceNameToSearchCursorUri.Keys)}"); + } + + return new EndpointConfiguration( + registrationCursorUri, + flatContainerCursorUri, + instanceNameToSearchConfig); + } + + private static Dictionary GetSuffixToUri(IDictionary arguments, string prefix) + { + var suffixToUri = new Dictionary(); + foreach (var key in arguments.Keys.Where(x => x.StartsWith(prefix))) + { + var suffix = key.Substring(prefix.Length); + suffixToUri[suffix] = arguments.GetOrThrow(key); + } + + return suffixToUri; + } + + public static IPackageMonitoringStatusService GetPackageMonitoringStatusService(IDictionary arguments, ICatalogStorageFactory storageFactory, ILoggerFactory loggerFactory) + { + return new PackageMonitoringStatusService( + new NamedStorageFactory(storageFactory, arguments.GetOrDefault(Arguments.PackageStatusFolder, Arguments.PackageStatusFolderDefault)), + loggerFactory.CreateLogger()); + } + + public static IStorageQueue CreateStorageQueue(IDictionary arguments, int version) + { + var storageType = arguments.GetOrThrow(Arguments.StorageType); + + if (Arguments.AzureStorageType.Equals(storageType, StringComparison.InvariantCultureIgnoreCase)) + { + var storageAccountName = arguments.GetOrThrow(Arguments.StorageAccountName); + var storageKeyValue = arguments.GetOrThrow(Arguments.StorageKeyValue); + var storageQueueName = arguments.GetOrDefault(Arguments.StorageQueueName); + + var credentials = new StorageCredentials(storageAccountName, storageKeyValue); + var account = new CloudStorageAccount(credentials, true); + return new StorageQueue(new AzureStorageQueue(account, storageQueueName), + new JsonMessageSerializer(JsonSerializerUtility.SerializerSettings), version); + } + else + { + throw new NotImplementedException("Only Azure storage queues are supported!"); + } + } + } +} \ No newline at end of file diff --git a/src/Ng/Db2Catalog.nuspec b/src/Ng/Db2Catalog.nuspec new file mode 100644 index 000000000..2869a5fe3 --- /dev/null +++ b/src/Ng/Db2Catalog.nuspec @@ -0,0 +1,16 @@ + + + + Db2Catalog + $version$ + .NET Foundation + .NET Foundation + The Db2Catalog job. + Copyright .NET Foundation + + + + + + + \ No newline at end of file diff --git a/src/Ng/Db2Monitoring.nuspec b/src/Ng/Db2Monitoring.nuspec new file mode 100644 index 000000000..e7dbabb60 --- /dev/null +++ b/src/Ng/Db2Monitoring.nuspec @@ -0,0 +1,16 @@ + + + + Db2Monitoring + $version$ + .NET Foundation + .NET Foundation + The Db2Monitoring job. + Copyright .NET Foundation + + + + + + + \ No newline at end of file diff --git a/src/Ng/Jobs/Catalog2DnxJob.cs b/src/Ng/Jobs/Catalog2DnxJob.cs new file mode 100644 index 000000000..3f112812c --- /dev/null +++ b/src/Ng/Jobs/Catalog2DnxJob.cs @@ -0,0 +1,123 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Protocol.Catalog; +using NuGet.Services.Configuration; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Dnx; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace Ng.Jobs +{ + public class Catalog2DnxJob : LoopingNgJob + { + private CommitCollector _collector; + private ReadWriteCursor _front; + private ReadCursor _back; + private Uri _destination; + + public Catalog2DnxJob(ILoggerFactory loggerFactory, + ITelemetryClient telemetryClient, + IDictionary telemetryGlobalDimensions) + : base(loggerFactory, telemetryClient, telemetryGlobalDimensions) + { + } + + public override string GetUsage() + { + return "Usage: ng catalog2dnx " + + $"-{Arguments.Source} " + + $"-{Arguments.ContentBaseAddress} " + + $"-{Arguments.StorageBaseAddress} " + + $"-{Arguments.StorageType} file|azure " + + $"[-{Arguments.StoragePath} ]" + + "|" + + $"[-{Arguments.StorageAccountName} " + + $"-{Arguments.StorageKeyValue} " + + $"-{Arguments.StorageContainer} " + + $"-{Arguments.StoragePath} " + + $"[-{Arguments.VaultName} " + + $"-{Arguments.UseManagedIdentity} true|false " + + $"-{Arguments.ClientId} Should not be set if {Arguments.UseManagedIdentity} is true" + + $"-{Arguments.CertificateThumbprint} Should not be set if {Arguments.UseManagedIdentity} is true" + + $"[-{Arguments.ValidateCertificate} true|false]]] " + + $"[-{Arguments.Verbose} true|false] " + + $"[-{Arguments.Interval} ]" + + $"[-{Arguments.HttpClientTimeoutInSeconds} ]" + + $"[-{Arguments.StorageSuffix} ]" + + $"[-{Arguments.PreferAlternatePackageSourceStorage} true|false " + + $"-{Arguments.StorageAccountNamePreferredPackageSourceStorage} " + + $"-{Arguments.StorageKeyValuePreferredPackageSourceStorage} " + + $"-{Arguments.StorageContainerPreferredPackageSourceStorage} " + + $"-{Arguments.StorageUseServerSideCopy} true|false]"; + } + + protected override void Init(IDictionary arguments, CancellationToken cancellationToken) + { + var source = arguments.GetOrThrow(Arguments.Source); + var verbose = arguments.GetOrDefault(Arguments.Verbose, false); + var contentBaseAddress = arguments.GetOrDefault(Arguments.ContentBaseAddress); + var storageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); + var httpClientTimeoutInSeconds = arguments.GetOrDefault(Arguments.HttpClientTimeoutInSeconds); + var httpClientTimeout = httpClientTimeoutInSeconds.HasValue ? (TimeSpan?)TimeSpan.FromSeconds(httpClientTimeoutInSeconds.Value) : null; + + StorageFactory preferredPackageSourceStorageFactory = null; + IAzureStorage preferredPackageSourceStorage = null; + + var preferAlternatePackageSourceStorage = arguments.GetOrDefault(Arguments.PreferAlternatePackageSourceStorage, defaultValue: false); + + if (preferAlternatePackageSourceStorage) + { + preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("PreferredPackageSourceStorage", arguments, verbose); + preferredPackageSourceStorage = preferredPackageSourceStorageFactory.Create() as IAzureStorage; + } + + Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"", + source, + storageFactory, + preferredPackageSourceStorageFactory); + Logger.LogInformation("HTTP client timeout: {Timeout}", httpClientTimeout); + + MaxDegreeOfParallelism = 256; + + _collector = new DnxCatalogCollector( + new Uri(source), + storageFactory, + preferredPackageSourceStorage, + contentBaseAddress == null ? null : new Uri(contentBaseAddress), + TelemetryService, + Logger, + MaxDegreeOfParallelism, + httpClient => new CatalogClient(new SimpleHttpClient(httpClient, LoggerFactory.CreateLogger()), LoggerFactory.CreateLogger()), + CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose), + httpClientTimeout); + + var storage = storageFactory.Create(); + _front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue); + _back = MemoryCursor.CreateMax(); + + _destination = storageFactory.BaseAddress; + TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = _destination.AbsoluteUri; + } + + protected override async Task RunInternalAsync(CancellationToken cancellationToken) + { + using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", _destination.AbsoluteUri)) + using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds)) + { + bool run; + do + { + run = await _collector.RunAsync(_front, _back, cancellationToken); + } + while (run); + } + } + } +} \ No newline at end of file diff --git a/src/Ng/Jobs/Catalog2IconJob.cs b/src/Ng/Jobs/Catalog2IconJob.cs new file mode 100644 index 000000000..4abd64856 --- /dev/null +++ b/src/Ng/Jobs/Catalog2IconJob.cs @@ -0,0 +1,104 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Protocol.Catalog; +using NuGet.Services.Configuration; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Icons; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace Ng.Jobs +{ + public class Catalog2IconJob : LoopingNgJob + { + private const int DegreeOfParallelism = 120; + private const string FailCacheTime = "failCacheTime"; + private IconsCollector _collector; + private DurableCursor _front; + + public Catalog2IconJob( + ILoggerFactory loggerFactory, + ITelemetryClient telemetryClient, + IDictionary telemetryGlobalDimensions) + : base(loggerFactory, telemetryClient, telemetryGlobalDimensions) + { + } + + protected override void Init(IDictionary arguments, CancellationToken cancellationToken) + { + ServicePointManager.DefaultConnectionLimit = DegreeOfParallelism; + + var verbose = arguments.GetOrDefault(Arguments.Verbose, false); + var packageStorageBase = arguments.GetOrThrow(Arguments.ContentBaseAddress); + var failCacheTime = arguments.GetOrDefault(FailCacheTime, TimeSpan.FromHours(1)); + var auxStorageFactory = CreateAuxStorageFactory(arguments, verbose); + var targetStorageFactory = CreateTargetStorageFactory(arguments, verbose); + var packageStorage = new AzureStorage( + storageBaseUri: new Uri(packageStorageBase), + maxExecutionTime: TimeSpan.FromMinutes(15), + serverTimeout: TimeSpan.FromMinutes(10), + useServerSideCopy: true, + compressContent: false, + verbose: true, + throttle: null); + var source = arguments.GetOrThrow(Arguments.Source); + var iconProcessor = new IconProcessor(TelemetryService, LoggerFactory.CreateLogger()); + var httpHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose); + var httpMessageHandler = httpHandlerFactory(); + var httpClient = new HttpClient(httpMessageHandler); + var simpleHttpClient = new SimpleHttpClient(httpClient, LoggerFactory.CreateLogger()); + var catalogClient = new CatalogClient(simpleHttpClient, LoggerFactory.CreateLogger()); + var httpResponseProvider = new HttpClientWrapper(httpClient); + var externalIconProvider = new ExternalIconContentProvider(httpResponseProvider, LoggerFactory.CreateLogger()); + var iconCopyResultCache = new IconCopyResultCache(auxStorageFactory.Create(), failCacheTime, LoggerFactory.CreateLogger()); + + var leafProcessor = new CatalogLeafDataProcessor( + packageStorage, + iconProcessor, + externalIconProvider, + iconCopyResultCache, + TelemetryService, + LoggerFactory.CreateLogger()); + + _collector = new IconsCollector( + new Uri(source), + TelemetryService, + targetStorageFactory, + catalogClient, + leafProcessor, + iconCopyResultCache, + auxStorageFactory, + CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose), + LoggerFactory.CreateLogger()); + var cursorStorage = auxStorageFactory.Create(); + _front = new DurableCursor(cursorStorage.ResolveUri("c2icursor.json"), cursorStorage, DateTime.MinValue.ToUniversalTime()); + } + + protected override async Task RunInternalAsync(CancellationToken cancellationToken) + { + bool run; + do + { + run = await _collector.RunAsync(_front, MemoryCursor.CreateMax(), cancellationToken); + } while (run); + } + + private IStorageFactory CreateAuxStorageFactory(IDictionary arguments, bool verbose) + { + return CommandHelpers.CreateSuffixedStorageFactory("Aux", arguments, verbose); + } + + private IStorageFactory CreateTargetStorageFactory(IDictionary arguments, bool verbose) + { + return CommandHelpers.CreateSuffixedStorageFactory("Target", arguments, verbose); + } + } +} diff --git a/src/Ng/Jobs/Catalog2MonitoringJob.cs b/src/Ng/Jobs/Catalog2MonitoringJob.cs new file mode 100644 index 000000000..0b2c3a3f1 --- /dev/null +++ b/src/Ng/Jobs/Catalog2MonitoringJob.cs @@ -0,0 +1,62 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.Configuration; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog.Monitoring; + +namespace Ng.Jobs +{ + /// + /// Runs a on the catalog. + /// The purpose of this job is to queue newly added, modified, or deleted packages for the to run validations on. + /// + public class Catalog2MonitoringJob : LoopingNgJob + { + private PackageValidatorContextEnqueuer _enqueuer; + + public Catalog2MonitoringJob( + ILoggerFactory loggerFactory, + ITelemetryClient telemetryClient, + IDictionary telemetryGlobalDimensions) + : base(loggerFactory, telemetryClient, telemetryGlobalDimensions) + { + } + + protected override void Init(IDictionary arguments, CancellationToken cancellationToken) + { + var source = arguments.GetOrThrow(Arguments.Source); + var verbose = arguments.GetOrDefault(Arguments.Verbose, false); + + CommandHelpers.AssertAzureStorage(arguments); + + var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); + var endpointConfiguration = CommandHelpers.GetEndpointConfiguration(arguments); + var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose); + var statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory); + var queue = CommandHelpers.CreateStorageQueue(arguments, PackageValidatorContext.Version); + + Logger.LogInformation( + "CONFIG storage: {Storage} registration cursor uri: {RegistrationCursorUri} flat-container cursor uri: {FlatContainerCursorUri}", + monitoringStorageFactory, endpointConfiguration.RegistrationCursorUri, endpointConfiguration.FlatContainerCursorUri); + + _enqueuer = ValidationFactory.CreatePackageValidatorContextEnqueuer( + queue, + source, + monitoringStorageFactory, + endpointConfiguration, + TelemetryService, + messageHandlerFactory, + LoggerFactory); + } + + protected override async Task RunInternalAsync(CancellationToken cancellationToken) + { + await _enqueuer.EnqueuePackageValidatorContexts(cancellationToken); + } + } +} \ No newline at end of file diff --git a/src/Ng/Jobs/Catalog2PackageFixupJob.cs b/src/Ng/Jobs/Catalog2PackageFixupJob.cs new file mode 100644 index 000000000..a6d86a2b0 --- /dev/null +++ b/src/Ng/Jobs/Catalog2PackageFixupJob.cs @@ -0,0 +1,178 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Auth; +using NuGet.Packaging.Core; +using NuGet.Services.Configuration; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog; + +namespace Ng.Jobs +{ + public class Catalog2PackageFixupJob : NgJob + { + private IServiceProvider _serviceProvider; + + public Catalog2PackageFixupJob( + ILoggerFactory loggerFactory, + ITelemetryClient telemetryClient, + IDictionary telemetryGlobalDimensions) + : base(loggerFactory, telemetryClient, telemetryGlobalDimensions) + { + ThreadPool.SetMinThreads(MaxDegreeOfParallelism, 4); + } + + public override string GetUsage() + { + return "Usage: ng catalog2packagefixup" + + $"-{Arguments.Source} " + + $"-{Arguments.Verify} true/false" + + $"-{Arguments.StorageAccountName} " + + $"-{Arguments.StorageKeyValue} " + + $"-{Arguments.StorageContainer} "; + } + + protected override void Init(IDictionary arguments, CancellationToken cancellationToken) + { + var source = arguments.GetOrThrow(Arguments.Source); + var storageAccount = arguments.GetOrThrow(Arguments.StorageAccountName); + var storageKey = arguments.GetOrThrow(Arguments.StorageKeyValue); + var storageContainer = arguments.GetOrThrow(Arguments.StorageContainer); + var verify = arguments.GetOrDefault(Arguments.Verify, false); + var verbose = arguments.GetOrDefault(Arguments.Verbose, false); + + var services = new ServiceCollection(); + + services.AddSingleton(new TelemetryService(TelemetryClient, GlobalTelemetryDimensions)); + services.AddSingleton(LoggerFactory); + services.AddLogging(); + + // Prepare the HTTP Client + services.AddSingleton(p => + { + var httpClient = new HttpClient(new WebRequestHandler + { + AllowPipelining = true + }); + + httpClient.DefaultRequestHeaders.Add("User-Agent", UserAgentUtility.GetUserAgent()); + httpClient.Timeout = TimeSpan.FromMinutes(10); + + return httpClient; + }); + + // Prepare the catalog reader. + services.AddSingleton(p => + { + var telemetryService = p.GetRequiredService(); + var httpMessageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(telemetryService, verbose); + + return new CollectorHttpClient(httpMessageHandlerFactory()); + }); + + services.AddTransient(p => + { + var collectorHttpClient = p.GetRequiredService(); + var telemetryService = p.GetRequiredService(); + + return new CatalogIndexReader(new Uri(source), collectorHttpClient, telemetryService); + }); + + // Prepare the Azure Blob Storage container. + services.AddSingleton(p => + { + var credentials = new StorageCredentials(storageAccount, storageKey); + var account = new CloudStorageAccount(credentials, useHttps: true); + + return account + .CreateCloudBlobClient() + .GetContainerReference(storageContainer); + }); + + // Prepare the handler that will run on each catalog entry. + if (verify) + { + Logger.LogInformation("Validating that all packages have the proper Content MD5 hash..."); + + services.AddTransient(); + } + else + { + Logger.LogInformation("Ensuring all packages have a Content MD5 hash..."); + + services.AddTransient(); + } + + services.AddTransient(); + + _serviceProvider = services.BuildServiceProvider(); + } + + protected override async Task RunInternalAsync(CancellationToken cancellationToken) + { + Logger.LogInformation("Parsing catalog for all entries."); + + var catalogReader = _serviceProvider.GetRequiredService(); + var entries = await catalogReader.GetEntries(); + + var latestEntries = entries + .GroupBy(c => new PackageIdentity(c.Id, c.Version)) + .Select(g => g.OrderByDescending(c => c.CommitTimeStamp).First()) + .Where(c => !c.IsDelete); + + var packageEntries = new ConcurrentBag(latestEntries); + + Logger.LogInformation("Processing packages."); + + var stopwatch = Stopwatch.StartNew(); + var processor = _serviceProvider.GetRequiredService(); + var totalEntries = packageEntries.Count; + + var tasks = Enumerable + .Range(0, MaxDegreeOfParallelism) + .Select(async i => + { + while (packageEntries.TryTake(out var entry)) + { + await processor.ProcessCatalogIndexEntryAsync(entry); + } + }) + .ToList(); + + tasks.Add(LogProgress(packageEntries)); + + await Task.WhenAll(tasks); + + Logger.LogInformation( + "Processed {ProcessedCount} packages in {ProcessDuration}", + totalEntries, + stopwatch.Elapsed); + } + + private async Task LogProgress(ConcurrentBag packageEntries) + { + int remaining; + do + { + await Task.Delay(TimeSpan.FromMinutes(1)); + + remaining = packageEntries.Count; + + Logger.LogInformation("{Remaining} packages left to enqueue...", remaining); + + } + while (remaining > 0); + } + } +} diff --git a/src/Ng/Jobs/Db2CatalogJob.cs b/src/Ng/Jobs/Db2CatalogJob.cs new file mode 100644 index 000000000..048f72a74 --- /dev/null +++ b/src/Ng/Jobs/Db2CatalogJob.cs @@ -0,0 +1,391 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Protocol; +using NuGet.Services.Configuration; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Services.Sql; +using Constants = NuGet.Services.Metadata.Catalog.Constants; + +namespace Ng.Jobs +{ + public class Db2CatalogJob : LoopingNgJob + { + protected bool Verbose; + protected ISqlConnectionFactory GalleryDbConnection; + protected PackageContentUriBuilder PackageContentUriBuilder; + protected IGalleryDatabaseQueryService GalleryDatabaseQueryService; + protected IStorage CatalogStorage; + protected IStorage AuditingStorage; + protected IStorage PreferredPackageSourceStorage; + protected DateTime? StartDate; + protected TimeSpan Timeout; + protected int Top; + protected Uri Destination; + protected bool SkipCreatedPackagesProcessing; + + public Db2CatalogJob( + ILoggerFactory loggerFactory, + ITelemetryClient telemetryClient, + IDictionary telemetryGlobalDimensions) + : base(loggerFactory, telemetryClient, telemetryGlobalDimensions) + { + } + + public override string GetUsage() + { + return "Usage: ng db2catalog " + + $"-{Arguments.ConnectionString} " + + $"-{Arguments.CursorSize} " + + $"-{Arguments.PackageContentUrlFormat} " + + $"-{Arguments.StorageBaseAddress} " + + $"-{Arguments.StorageType} file|azure " + + $"[-{Arguments.StoragePath} ]" + + "|" + + $"[-{Arguments.StorageAccountName} " + + $"-{Arguments.StorageKeyValue} " + + $"-{Arguments.StorageContainer} " + + $"-{Arguments.StoragePath} " + + $"[-{Arguments.VaultName} " + + $"-{Arguments.UseManagedIdentity} true|false " + + $"-{Arguments.ClientId} Should not be set if {Arguments.UseManagedIdentity} is true" + + $"-{Arguments.CertificateThumbprint} Should not be set if {Arguments.UseManagedIdentity} is true" + + $"[-{Arguments.ValidateCertificate} true|false]]] " + + $"-{Arguments.StorageTypeAuditing} file|azure " + + $"[-{Arguments.StoragePathAuditing} ]" + + "|" + + $"[-{Arguments.StorageAccountNameAuditing} " + + $"-{Arguments.StorageKeyValueAuditing} " + + $"-{Arguments.StorageContainerAuditing} " + + $"-{Arguments.StoragePathAuditing} ] " + + "|" + + $"[-{Arguments.PreferAlternatePackageSourceStorage} true|false " + + $"-{Arguments.StorageAccountNamePreferredPackageSourceStorage} " + + $"-{Arguments.StorageKeyValuePreferredPackageSourceStorage} " + + $"-{Arguments.StorageContainerPreferredPackageSourceStorage} ] " + + $"[-{Arguments.SkipCreatedPackagesProcessing} true|false] " + + $"[-{Arguments.Verbose} true|false] " + + $"[-{Arguments.Interval} ] " + + $"[-{Arguments.StartDate} ]"; + } + + protected override void Init(IDictionary arguments, CancellationToken cancellationToken) + { + Verbose = arguments.GetOrDefault(Arguments.Verbose, false); + StartDate = arguments.GetOrDefault(Arguments.StartDate, Constants.DateTimeMinValueUtc); + Top = arguments.GetOrDefault(Arguments.CursorSize, 20); + SkipCreatedPackagesProcessing = arguments.GetOrDefault(Arguments.SkipCreatedPackagesProcessing, false); + + StorageFactory preferredPackageSourceStorageFactory = null; + + var preferAlternatePackageSourceStorage = arguments.GetOrDefault(Arguments.PreferAlternatePackageSourceStorage, false); + + if (preferAlternatePackageSourceStorage) + { + preferredPackageSourceStorageFactory = CommandHelpers.CreateSuffixedStorageFactory( + "PreferredPackageSourceStorage", + arguments, + Verbose, + new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit))); + } + + var catalogStorageFactory = CommandHelpers.CreateStorageFactory( + arguments, + Verbose, + new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit))); + + var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory( + "Auditing", + arguments, + Verbose, + new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit))); + + Logger.LogInformation("CONFIG source: \"{ConfigSource}\" storage: \"{Storage}\" preferred package source storage: \"{PreferredPackageSourceStorage}\"", + GalleryDbConnection, + catalogStorageFactory, + preferredPackageSourceStorageFactory); + + CatalogStorage = catalogStorageFactory.Create(); + AuditingStorage = auditingStorageFactory.Create(); + + if (preferAlternatePackageSourceStorage) + { + PreferredPackageSourceStorage = preferredPackageSourceStorageFactory.Create(); + } + + Destination = catalogStorageFactory.BaseAddress; + TelemetryService.GlobalDimensions[TelemetryConstants.Destination] = Destination.AbsoluteUri; + + // Setup gallery database access + PackageContentUriBuilder = new PackageContentUriBuilder( + arguments.GetOrThrow(Arguments.PackageContentUrlFormat)); + + var connectionString = arguments.GetOrThrow(Arguments.ConnectionString); + GalleryDbConnection = new AzureSqlConnectionFactory( + connectionString, + SecretInjector, + LoggerFactory.CreateLogger()); + + var timeoutInSeconds = arguments.GetOrDefault(Arguments.SqlCommandTimeoutInSeconds, 300); + Timeout = TimeSpan.FromSeconds(timeoutInSeconds); + GalleryDatabaseQueryService = new GalleryDatabaseQueryService( + GalleryDbConnection, + PackageContentUriBuilder, + TelemetryService, + timeoutInSeconds); + } + + protected override async Task RunInternalAsync(CancellationToken cancellationToken) + { + using (Logger.BeginScope($"Logging for {{{TelemetryConstants.Destination}}}", Destination.AbsoluteUri)) + using (TelemetryService.TrackDuration(TelemetryConstants.JobLoopSeconds)) + using (var client = CreateHttpClient()) + { + uint packagesDeleted; + uint packagesCreated; + uint packagesEdited; + + client.Timeout = Timeout; + + var packageCatalogItemCreator = PackageCatalogItemCreator.Create( + client, + TelemetryService, + Logger, + PreferredPackageSourceStorage); + + do + { + packagesDeleted = 0; + packagesCreated = 0; + packagesEdited = 0; + + // baseline timestamps + var catalogProperties = await CatalogProperties.ReadAsync(CatalogStorage, TelemetryService, cancellationToken); + var lastCreated = catalogProperties.LastCreated ?? (StartDate ?? Constants.DateTimeMinValueUtc); + var lastEdited = catalogProperties.LastEdited ?? lastCreated; + var lastDeleted = catalogProperties.LastDeleted ?? lastCreated; + + if (lastDeleted == Constants.DateTimeMinValueUtc) + { + lastDeleted = SkipCreatedPackagesProcessing ? lastEdited : lastCreated; + } + + try + { + if (lastDeleted > Constants.DateTimeMinValueUtc) + { + using (TelemetryService.TrackDuration(TelemetryConstants.DeletedPackagesSeconds)) + { + Logger.LogInformation("CATALOG LastDeleted: {CatalogDeletedTime}", lastDeleted.ToString("O")); + + var deletedPackages = await GetDeletedPackages(AuditingStorage, lastDeleted); + + packagesDeleted = (uint)deletedPackages.SelectMany(x => x.Value).Count(); + Logger.LogInformation("FEED DeletedPackages: {DeletedPackagesCount}", packagesDeleted); + + // We want to ensure a commit only contains each package once at most. + // Therefore we segment by package id + version. + var deletedPackagesSegments = SegmentPackageDeletes(deletedPackages); + foreach (var deletedPackagesSegment in deletedPackagesSegments) + { + lastDeleted = await Deletes2Catalog( + deletedPackagesSegment, CatalogStorage, lastCreated, lastEdited, lastDeleted, cancellationToken); + + // Wait for one second to ensure the next catalog commit gets a new timestamp + Thread.Sleep(TimeSpan.FromSeconds(1)); + } + } + } + + if (!SkipCreatedPackagesProcessing) + { + using (TelemetryService.TrackDuration(TelemetryConstants.CreatedPackagesSeconds)) + { + Logger.LogInformation("CATALOG LastCreated: {CatalogLastCreatedTime}", lastCreated.ToString("O")); + + var createdPackages = await GalleryDatabaseQueryService.GetPackagesCreatedSince(lastCreated, Top); + + packagesCreated = (uint)createdPackages.SelectMany(x => x.Value).Count(); + Logger.LogInformation("DATABASE CreatedPackages: {CreatedPackagesCount}", packagesCreated); + + lastCreated = await CatalogWriterHelper.WritePackageDetailsToCatalogAsync( + packageCatalogItemCreator, + createdPackages, + CatalogStorage, + lastCreated, + lastEdited, + lastDeleted, + MaxDegreeOfParallelism, + createdPackages: true, + updateCreatedFromEdited: false, + cancellationToken: cancellationToken, + telemetryService: TelemetryService, + logger: Logger); + } + } + + using (TelemetryService.TrackDuration(TelemetryConstants.EditedPackagesSeconds)) + { + Logger.LogInformation("CATALOG LastEdited: {CatalogLastEditedTime}", lastEdited.ToString("O")); + + var editedPackages = await GalleryDatabaseQueryService.GetPackagesEditedSince(lastEdited, Top); + + packagesEdited = (uint)editedPackages.SelectMany(x => x.Value).Count(); + Logger.LogInformation("DATABASE EditedPackages: {EditedPackagesCount}", packagesEdited); + + lastEdited = await CatalogWriterHelper.WritePackageDetailsToCatalogAsync( + packageCatalogItemCreator, + editedPackages, + CatalogStorage, + lastCreated, + lastEdited, + lastDeleted, + MaxDegreeOfParallelism, + createdPackages: false, + updateCreatedFromEdited: SkipCreatedPackagesProcessing, + cancellationToken: cancellationToken, + telemetryService: TelemetryService, + logger: Logger); + } + } + finally + { + TelemetryService.TrackMetric(TelemetryConstants.DeletedPackagesCount, packagesDeleted); + + if (!SkipCreatedPackagesProcessing) + { + TelemetryService.TrackMetric(TelemetryConstants.CreatedPackagesCount, packagesCreated); + } + + TelemetryService.TrackMetric(TelemetryConstants.EditedPackagesCount, packagesEdited); + } + } while (packagesDeleted > 0 || packagesCreated > 0 || packagesEdited > 0); + } + } + + // Wrapper function for CatalogUtility.CreateHttpClient + // Overriden by NgTests.TestableDb2CatalogJob + protected virtual HttpClient CreateHttpClient() + { + return FeedHelpers.CreateHttpClient(CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, Verbose)); + } + + private async Task>> GetDeletedPackages(IStorage auditingStorage, DateTime since) + { + var result = new SortedList>(); + + // Get all audit blobs (based on their filename which starts with a date that can be parsed) + // NOTE we're getting more files than needed (to account for a time difference between servers) + var minimumFileTime = since.AddMinutes(-15); + var auditEntries = await DeletionAuditEntry.GetAsync(auditingStorage, CancellationToken.None, + minTime: minimumFileTime, logger: Logger); + + foreach (var auditEntry in auditEntries) + { + if (!string.IsNullOrEmpty(auditEntry.PackageId) && !string.IsNullOrEmpty(auditEntry.PackageVersion) && auditEntry.TimestampUtc > since) + { + // Mark the package "deleted" + if (!result.TryGetValue(auditEntry.TimestampUtc.Value, out var packages)) + { + packages = new List(); + result.Add(auditEntry.TimestampUtc.Value, packages); + } + + packages.Add(new FeedPackageIdentity(auditEntry.PackageId, auditEntry.PackageVersion)); + } + } + + return result; + } + + private static IEnumerable>> SegmentPackageDeletes(SortedList> packageDeletes) + { + var packageIdentityTracker = new HashSet(); + var currentSegment = new SortedList>(); + foreach (var entry in packageDeletes) + { + if (!currentSegment.ContainsKey(entry.Key)) + { + currentSegment.Add(entry.Key, new List()); + } + + var curentSegmentPackages = currentSegment[entry.Key]; + foreach (var packageIdentity in entry.Value) + { + var key = packageIdentity.Id + "|" + packageIdentity.Version; + if (packageIdentityTracker.Contains(key)) + { + // Duplicate, return segment + yield return currentSegment; + + // Clear current segment + currentSegment.Clear(); + currentSegment.Add(entry.Key, new List()); + curentSegmentPackages = currentSegment[entry.Key]; + packageIdentityTracker.Clear(); + } + + // Add to segment + curentSegmentPackages.Add(packageIdentity); + packageIdentityTracker.Add(key); + } + } + + if (currentSegment.Any()) + { + yield return currentSegment; + } + } + + private async Task Deletes2Catalog( + SortedList> packages, + IStorage storage, + DateTime lastCreated, + DateTime lastEdited, + DateTime lastDeleted, + CancellationToken cancellationToken) + { + var writer = new AppendOnlyCatalogWriter( + storage, + TelemetryService, + Constants.MaxPageSize); + + if (packages == null || packages.Count == 0) + { + return lastDeleted; + } + + foreach (var entry in packages) + { + foreach (var packageIdentity in entry.Value) + { + var catalogItem = new DeleteCatalogItem(packageIdentity.Id, packageIdentity.Version, entry.Key); + writer.Add(catalogItem); + + Logger.LogInformation("Delete: {PackageId} {PackageVersion}", packageIdentity.Id, packageIdentity.Version); + } + + lastDeleted = entry.Key; + } + + var commitMetadata = PackageCatalog.CreateCommitMetadata(writer.RootUri, new CommitMetadata(lastCreated, lastEdited, lastDeleted)); + + await writer.Commit(commitMetadata, cancellationToken); + + Logger.LogInformation("COMMIT package deletes to catalog."); + + return lastDeleted; + } + } +} \ No newline at end of file diff --git a/src/Ng/Jobs/Db2MonitoringJob.cs b/src/Ng/Jobs/Db2MonitoringJob.cs new file mode 100644 index 000000000..3e20e6aca --- /dev/null +++ b/src/Ng/Jobs/Db2MonitoringJob.cs @@ -0,0 +1,213 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Protocol; +using NuGet.Services.Configuration; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Services.Metadata.Catalog.Monitoring.Monitoring; +using NuGet.Services.Sql; +using NuGet.Services.Storage; +using CatalogStorage = NuGet.Services.Metadata.Catalog.Persistence.Storage; +using CatalogStorageFactory = NuGet.Services.Metadata.Catalog.Persistence.StorageFactory; +using Constants = NuGet.Services.Metadata.Catalog.Constants; + +namespace Ng.Jobs +{ + public class Db2MonitoringJob : LoopingNgJob + { + /// + /// This job will continuously check packages within a range of minus 2 * and minus 1 * . + /// + private readonly static TimeSpan ReprocessRange = TimeSpan.FromHours(1); + + /// + /// Any values greater than will be ignored by . + /// + private const int Top = Constants.MaxPageSize; + private const string GalleryCursorFileName = "gallerycursor.json"; + private const string DeletedCursorFileName = "deletedcursor.json"; + + private const int DefaultMaxQueueSize = 100; + private int _maxRequeueQueueSize; + + private IGalleryDatabaseQueryService _galleryDatabaseQueryService; + private IStorageQueue _packageValidatorContextQueue; + private IPackageMonitoringStatusService _statusService; + private ReadCursor _monitoringCursor; + private ReadWriteCursor _galleryCursor; + + private CatalogStorage _auditingStorage; + private ReadWriteCursor _deletedCursor; + + private CollectorHttpClient _client; + + public Db2MonitoringJob( + ILoggerFactory loggerFactory, + ITelemetryClient telemetryClient, + IDictionary telemetryGlobalDimensions) + : base(loggerFactory, telemetryClient, telemetryGlobalDimensions) + { + } + + protected override void Init(IDictionary arguments, CancellationToken cancellationToken) + { + var verbose = arguments.GetOrDefault(Arguments.Verbose, false); + _maxRequeueQueueSize = arguments.GetOrDefault(Arguments.MaxRequeueQueueSize, DefaultMaxQueueSize); + + CommandHelpers.AssertAzureStorage(arguments); + + var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); + _statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory); + _packageValidatorContextQueue = CommandHelpers.CreateStorageQueue(arguments, PackageValidatorContext.Version); + + Logger.LogInformation( + "CONFIG storage: {Storage}", + monitoringStorageFactory); + + _monitoringCursor = ValidationFactory.GetFront(monitoringStorageFactory); + _galleryCursor = CreateCursor(monitoringStorageFactory, GalleryCursorFileName); + _deletedCursor = CreateCursor(monitoringStorageFactory, DeletedCursorFileName); + + var connectionString = arguments.GetOrThrow(Arguments.ConnectionString); + var galleryDbConnection = new AzureSqlConnectionFactory( + connectionString, + SecretInjector, + LoggerFactory.CreateLogger()); + + var packageContentUriBuilder = new PackageContentUriBuilder( + arguments.GetOrThrow(Arguments.PackageContentUrlFormat)); + + var timeoutInSeconds = arguments.GetOrDefault(Arguments.SqlCommandTimeoutInSeconds, 300); + _galleryDatabaseQueryService = new GalleryDatabaseQueryService( + galleryDbConnection, + packageContentUriBuilder, + TelemetryService, + timeoutInSeconds); + + var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory( + "Auditing", + arguments, + verbose, + new SemaphoreSlimThrottle(new SemaphoreSlim(ServicePointManager.DefaultConnectionLimit))); + + _auditingStorage = auditingStorageFactory.Create(); + + var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose); + _client = new CollectorHttpClient(messageHandlerFactory()); + } + + protected override async Task RunInternalAsync(CancellationToken cancellationToken) + { + var databaseSource = new DatabasePackageStatusOutdatedCheckSource( + _galleryCursor, _galleryDatabaseQueryService); + + var auditingSource = new AuditingStoragePackageStatusOutdatedCheckSource( + _deletedCursor, _auditingStorage, LoggerFactory.CreateLogger()); + + var sources = new IPackageStatusOutdatedCheckSource[] { databaseSource, auditingSource }; + + var hasPackagesToProcess = true; + while (hasPackagesToProcess) + { + var currentMessageCount = await _packageValidatorContextQueue.GetMessageCount(cancellationToken); + if (currentMessageCount > _maxRequeueQueueSize) + { + Logger.LogInformation( + "Can't continue processing packages because the queue has too many messages ({CurrentMessageCount} > {MaxRequeueQueueSize})!", + currentMessageCount, _maxRequeueQueueSize); + return; + } + + hasPackagesToProcess = await CheckPackages( + sources, + cancellationToken); + } + + Logger.LogInformation("All packages have had their status checked."); + await _monitoringCursor.LoadAsync(cancellationToken); + var newCursorValue = _monitoringCursor.Value - ReprocessRange - ReprocessRange; + Logger.LogInformation("Restarting source cursors to {NewCursorValue}.", newCursorValue); + foreach (var source in sources) + { + await source.MoveBackAsync(newCursorValue, cancellationToken); + } + } + + private async Task CheckPackages( + IReadOnlyCollection sources, + CancellationToken cancellationToken) + { + Logger.LogInformation("Fetching packages to check status of."); + var packagesToCheck = new List(); + await _monitoringCursor.LoadAsync(cancellationToken); + foreach (var source in sources) + { + packagesToCheck.AddRange(await source.GetPackagesToCheckAsync( + _monitoringCursor.Value - ReprocessRange, Top, cancellationToken)); + } + + var packagesToCheckBag = new ConcurrentBag(packagesToCheck); + + Logger.LogInformation("Found {PackagesToCheckCount} packages to check status of.", packagesToCheck.Count()); + await ParallelAsync.Repeat(() => ProcessPackagesAsync(packagesToCheckBag, cancellationToken)); + Logger.LogInformation("Finished checking status of packages."); + + foreach (var source in sources) + { + await source.MarkPackagesCheckedAsync(cancellationToken); + } + + return packagesToCheck.Any(); + } + + private async Task ProcessPackagesAsync( + ConcurrentBag checkBag, CancellationToken cancellationToken) + { + while (checkBag.TryTake(out var check)) + { + if (await IsStatusOutdatedAsync(check, cancellationToken)) + { + Logger.LogWarning("Status for {Id} {Version} is outdated!", check.Identity.Id, check.Identity.Version); + var context = new PackageValidatorContext(check.Identity, null); + await _packageValidatorContextQueue.AddAsync(context, cancellationToken); + } + else + { + Logger.LogInformation("Status for {Id} {Version} is up to date.", check.Identity.Id, check.Identity.Version); + } + } + } + + private async Task IsStatusOutdatedAsync( + PackageStatusOutdatedCheck check, CancellationToken cancellationToken) + { + var status = await _statusService.GetAsync(check.Identity, cancellationToken); + + var catalogEntries = status?.ValidationResult?.CatalogEntries; + if (catalogEntries == null || !catalogEntries.Any()) + { + return true; + } + + var latestCatalogEntryTimestampMetadata = await PackageTimestampMetadata.FromCatalogEntries(_client, catalogEntries); + return check.Timestamp > latestCatalogEntryTimestampMetadata.Last; + } + + private DurableCursor CreateCursor(CatalogStorageFactory storageFactory, string filename) + { + var storage = storageFactory.Create(); + return new DurableCursor(storage.ResolveUri(filename), storage, MemoryCursor.MinValue); + } + } +} diff --git a/src/Ng/Jobs/LightningJob.cs b/src/Ng/Jobs/LightningJob.cs new file mode 100644 index 000000000..5c8f570a2 --- /dev/null +++ b/src/Ng/Jobs/LightningJob.cs @@ -0,0 +1,634 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Autofac; +using Autofac.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage.Blob; +using NuGet.Jobs.Catalog2Registration; +using NuGet.Packaging.Core; +using NuGet.Protocol.Catalog; +using NuGet.Services; +using NuGet.Services.Configuration; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGetGallery; + +namespace Ng.Jobs +{ + public class LightningJob : NgJob + { + private const string JsonDriver = "json"; + + public LightningJob( + ILoggerFactory loggerFactory, + ITelemetryClient telemetryClient, + IDictionary telemetryGlobalDimensions) + : base(loggerFactory, telemetryClient, telemetryGlobalDimensions) + { + } + + private static void PrintLightning() + { + var currentColor = Console.ForegroundColor; + + Console.ForegroundColor = ConsoleColor.Yellow; + Console.WriteLine(" ,/"); + Console.WriteLine(" ,'/"); + Console.WriteLine(" ,' /"); + Console.WriteLine(" ,' /_____,"); + Console.WriteLine(" .'____ ,' NuGet - ng.exe lightning"); + Console.WriteLine(" / ,'"); + Console.WriteLine(" / ,' The lightning fast catalog2registration."); + Console.WriteLine(" /,'"); + Console.WriteLine(" /'"); + Console.ForegroundColor = currentColor; + Console.WriteLine(); + } + + public override string GetUsage() + { + var sw = new StringWriter(); + + sw.WriteLine($"Usage: ng lightning -{Arguments.Command} prepare|strike"); + sw.WriteLine(); + sw.WriteLine($"The following arguments are supported on both lightning commands."); + sw.WriteLine(); + sw.WriteLine($" -{Arguments.ContentBaseAddress} "); + sw.WriteLine($" The base address for package contents."); + sw.WriteLine($" -{Arguments.GalleryBaseAddress} "); + sw.WriteLine($" The base address for gallery."); + sw.WriteLine($" -{Arguments.FlatContainerName} "); + sw.WriteLine($" The storage container name for the flat container resource."); + sw.WriteLine($" -{Arguments.StorageSuffix} "); + sw.WriteLine($" String to indicate the target storage suffix. If china for example core.chinacloudapi.cn needs to be used."); + sw.WriteLine($" The default value is 'core.windows.net'."); + sw.WriteLine($" -{Arguments.Verbose} true|false"); + sw.WriteLine($" Switch output verbosity on/off."); + sw.WriteLine($" The default value is false."); + sw.WriteLine(); + sw.WriteLine($" -{Arguments.StorageBaseAddress} "); + sw.WriteLine($" Base address to write into registration blobs."); + sw.WriteLine($" -{Arguments.StorageAccountName} "); + sw.WriteLine($" Azure Storage account name."); + sw.WriteLine($" -{Arguments.StorageKeyValue} "); + sw.WriteLine($" Azure Storage account name."); + sw.WriteLine($" -{Arguments.StorageContainer} "); + sw.WriteLine($" Container to generate registrations in."); + sw.WriteLine(); + sw.WriteLine($" -{Arguments.CompressedStorageBaseAddress} "); + sw.WriteLine($" Compressed only: Base address to write into registration blobs."); + sw.WriteLine($" -{Arguments.CompressedStorageAccountName} "); + sw.WriteLine($" Compressed only: Azure Storage account name."); + sw.WriteLine($" -{Arguments.CompressedStorageKeyValue} "); + sw.WriteLine($" Compressed only: Azure Storage account name."); + sw.WriteLine($" -{Arguments.CompressedStorageContainer} "); + sw.WriteLine($" Compressed only: Container to generate registrations in."); + sw.WriteLine(); + sw.WriteLine($" -{Arguments.SemVer2StorageBaseAddress} "); + sw.WriteLine($" SemVer 2.0.0 only: Base address to write into registration blobs."); + sw.WriteLine($" -{Arguments.SemVer2StorageAccountName} "); + sw.WriteLine($" SemVer 2.0.0 only: Azure Storage account name."); + sw.WriteLine($" -{Arguments.SemVer2StorageKeyValue} "); + sw.WriteLine($" SemVer 2.0.0 only: Azure Storage account name for SemVer 2.0.0."); + sw.WriteLine($" -{Arguments.SemVer2StorageContainer} "); + sw.WriteLine($" SemVer 2.0.0 only: Container to generate registrations in."); + sw.WriteLine(); + sw.WriteLine($"The prepare command:"); + sw.WriteLine($" ng lightning -{Arguments.Command} prepare ..."); + sw.WriteLine(); + sw.WriteLine($" -{Arguments.OutputFolder} "); + sw.WriteLine($" The folder to generate files in."); + sw.WriteLine($" -{Arguments.TemplateFile} "); + sw.WriteLine($" The lightning-template.txt that calls the strike command per batch."); + sw.WriteLine($" This file can be found in Ng source directory."); + sw.WriteLine($" -{Arguments.Source} "); + sw.WriteLine($" The catalog index.json URL to work with."); + sw.WriteLine($" -{Arguments.BatchSize} "); + sw.WriteLine($" The batch size."); + sw.WriteLine(); + sw.WriteLine($"Traverses the given catalog and, using a template file and batch size,"); + sw.WriteLine($"generates executable commands that can be run in parallel."); + sw.WriteLine($"The generated index.txt contains an alphabetical listing of all packages"); + sw.WriteLine($"in the catalog with their entries."); + sw.WriteLine(); + sw.WriteLine($"The strike command:"); + sw.WriteLine($" ng lightning -{Arguments.Command} strike ..."); + sw.WriteLine(); + sw.WriteLine($" -{Arguments.IndexFile} "); + sw.WriteLine($" Index file generated by the lightning prepare command."); + sw.WriteLine($" -{Arguments.CursorFile} "); + sw.WriteLine($" Cursor file containing range of the batch."); + sw.WriteLine(); + sw.WriteLine($"The lightning strike command is used by the batch files generated with"); + sw.WriteLine($"the prepare command. It creates registrations for a given batch of catalog"); + sw.WriteLine($"entries."); + + return sw.ToString(); + } + + #region Shared Arguments + private string _command; + private bool _verbose; + private TextWriter _log; + private IDictionary _arguments; + #endregion + + #region Prepare Arguments + private string _outputFolder; + private string _catalogIndex; + private string _templateFile; + private string _batchSize; + #endregion + + #region Strike Arguments + private string _indexFile; + private string _cursorFile; + #endregion + + protected override void Init(IDictionary arguments, CancellationToken cancellationToken) + { + PrintLightning(); + + // Hard code against Azure storage. + arguments[Arguments.StorageType] = Arguments.AzureStorageType; + + _command = arguments.GetOrThrow(Arguments.Command); + _verbose = arguments.GetOrDefault(Arguments.Verbose, false); + _log = _verbose ? Console.Out : new StringWriter(); + // We save the arguments because the "prepare" command generates "strike" commands. Some of the arguments + // used by "prepare" should be used when executing "strike". + _arguments = arguments; + + switch (_command.ToLowerInvariant()) + { + case "charge": + case "prepare": + InitPrepare(arguments); + break; + case "strike": + InitStrike(arguments); + break; + default: + throw new NotSupportedException($"The lightning command '{_command}' is not supported."); + } + } + + private void InitPrepare(IDictionary arguments) + { + _outputFolder = arguments.GetOrThrow(Arguments.OutputFolder); + _catalogIndex = arguments.GetOrThrow(Arguments.Source); + _templateFile = arguments.GetOrThrow(Arguments.TemplateFile); + _batchSize = arguments.GetOrThrow(Arguments.BatchSize); + } + + private void InitStrike(IDictionary arguments) + { + _indexFile = arguments.GetOrThrow(Arguments.IndexFile); + _cursorFile = arguments.GetOrThrow(Arguments.CursorFile); + } + + protected override async Task RunInternalAsync(CancellationToken cancellationToken) + { + switch (_command.ToLowerInvariant()) + { + case "charge": + case "prepare": + await PrepareAsync(); + break; + case "strike": + await StrikeAsync(); + break; + default: + throw new ArgumentNullException(); + } + } + + private async Task PrepareAsync() + { + _log.WriteLine("Making sure folder {0} exists.", _outputFolder); + if (!Directory.Exists(_outputFolder)) + { + Directory.CreateDirectory(_outputFolder); + } + + // Create reindex file + _log.WriteLine("Start preparing lightning reindex file..."); + + var latestCommit = DateTime.MinValue; + int numberOfEntries = 0; + string indexFile = Path.Combine(_outputFolder, "index.txt"); + string optionalArgumentsTemplate = "optionalArguments"; + + using (var streamWriter = new StreamWriter(indexFile, false)) + { + var httpMessageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, _verbose); + var collectorHttpClient = new CollectorHttpClient(httpMessageHandlerFactory()); + var catalogIndexReader = new CatalogIndexReader(new Uri(_catalogIndex), collectorHttpClient, TelemetryService); + + var catalogIndexEntries = await catalogIndexReader.GetEntries(); + + foreach (var packageRegistrationGroup in catalogIndexEntries + .OrderBy(x => x.CommitTimeStamp) + .ThenBy(x => x.Id, StringComparer.OrdinalIgnoreCase) + .ThenBy(x => x.Version) + .GroupBy(x => x.Id, StringComparer.OrdinalIgnoreCase)) + { + streamWriter.WriteLine("Element@{0}. {1}", numberOfEntries++, packageRegistrationGroup.Key); + + var latestCatalogPages = new Dictionary(); + + foreach (CatalogIndexEntry catalogIndexEntry in packageRegistrationGroup) + { + string key = catalogIndexEntry.Version.ToNormalizedString(); + if (latestCatalogPages.ContainsKey(key)) + { + latestCatalogPages[key] = catalogIndexEntry.Uri; + } + else + { + latestCatalogPages.Add(key, catalogIndexEntry.Uri); + } + + if (latestCommit < catalogIndexEntry.CommitTimeStamp) + { + latestCommit = catalogIndexEntry.CommitTimeStamp; + } + } + + foreach (var latestCatalogPage in latestCatalogPages) + { + streamWriter.WriteLine("{0}", latestCatalogPage.Value); + } + } + } + + _log.WriteLine("Finished preparing lightning reindex file. Output file: {0}", indexFile); + + // Create the containers + _log.WriteLine("Creating the containers..."); + var container = GetAutofacContainer(); + var blobClient = container.Resolve(); + var config = container.Resolve>().Value; + foreach (var name in new[] { config.LegacyStorageContainer, config.GzippedStorageContainer, config.SemVer2StorageContainer }) + { + var reference = blobClient.GetContainerReference(name); + await reference.CreateIfNotExistAsync(); + await reference.SetPermissionsAsync(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob }); + } + + // Write cursor to storage + _log.WriteLine("Start writing new cursor..."); + var storageFactory = container.ResolveKeyed(DependencyInjectionExtensions.CursorBindingKey); + var storage = storageFactory.Create(); + var cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, latestCommit) + { + Value = latestCommit + }; + + await cursor.SaveAsync(CancellationToken.None); + _log.WriteLine("Finished writing new cursor."); + + // Write command files + _log.WriteLine("Start preparing lightning reindex command files..."); + + string templateFileContents; + using (var templateStreamReader = new StreamReader(_templateFile)) + { + templateFileContents = await templateStreamReader.ReadToEndAsync(); + } + + int batchNumber = 0; + int batchSizeValue = int.Parse(_batchSize); + for (int batchStart = 0; batchStart < numberOfEntries; batchStart += batchSizeValue) + { + var batchEnd = (batchStart + batchSizeValue - 1); + if (batchEnd >= numberOfEntries) + { + batchEnd = numberOfEntries - 1; + } + + var cursorCommandFileName = "cursor" + batchNumber + ".cmd"; + var cursorTextFileName = "cursor" + batchNumber + ".txt"; + + using (var cursorCommandStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorCommandFileName))) + using (var cursorTextStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorTextFileName))) + { + var commandStreamContents = templateFileContents; + + var replacements = _arguments + .Concat(new[] + { + new KeyValuePair("indexFile", indexFile), + new KeyValuePair("cursorFile", cursorTextFileName) + }); + + foreach (var replacement in replacements) + { + commandStreamContents = commandStreamContents + .Replace($"[{replacement.Key}]", replacement.Value); + } + + //the not required arguments need to be added only if they were passed in + //they cannot be hardcoded in the template + var optionalArguments = new StringBuilder(); + AppendOptionalArgument(optionalArguments, Arguments.FlatContainerName); + AppendOptionalArgument(optionalArguments, Arguments.StorageSuffix); + AppendOptionalArgument(optionalArguments, Arguments.Verbose); + + commandStreamContents = commandStreamContents + .Replace($"[{optionalArgumentsTemplate}]", optionalArguments.ToString()); + + await cursorCommandStreamWriter.WriteLineAsync(commandStreamContents); + await cursorTextStreamWriter.WriteLineAsync(batchStart + "," + batchEnd); + } + + batchNumber++; + } + + _log.WriteLine("Finished preparing lightning reindex command files."); + + _log.WriteLine("You can now copy the {0} file and all cursor*.cmd, cursor*.txt", indexFile); + _log.WriteLine("to multiple machines and run the cursor*.cmd files in parallel."); + } + + private void AppendOptionalArgument(StringBuilder optionalArguments, string name) + { + if (_arguments.ContainsKey(name)) + { + if (optionalArguments.Length > 0) + { + optionalArguments.AppendLine(" ^"); + optionalArguments.Append(" "); + } + + optionalArguments.AppendFormat("-{0} {1}", name, _arguments[name]); + } + } + + private async Task StrikeAsync() + { + _log.WriteLine("Start lightning strike for {0}...", _cursorFile); + + // Get batch range + int batchStart; + int batchEnd; + using (var cursorStreamReader = new StreamReader(_cursorFile)) + { + var batchRange = (await cursorStreamReader.ReadLineAsync()).Split(','); + batchStart = int.Parse(batchRange[0]); + batchEnd = int.Parse(batchRange[1]); + + _log.WriteLine("Batch range: {0} - {1}", batchStart, batchEnd); + } + if (batchStart > batchEnd) + { + _log.WriteLine("Batch already finished."); + return; + } + + // Time to strike + var container = GetAutofacContainer(); + var catalogClient = container.Resolve(); + var registrationUpdater = container.Resolve(); + + var startElement = string.Format("Element@{0}.", batchStart); + var endElement = string.Format("Element@{0}.", batchEnd + 1); + using (var indexStreamReader = new StreamReader(_indexFile)) + { + string line; + + // Skip entries that are not in the current batch bounds + do + { + line = await indexStreamReader.ReadLineAsync(); + } + while (!line.Contains(startElement)); + + // Run until we're outside the current batch bounds + while (!string.IsNullOrEmpty(line) && !line.Contains(endElement) && !indexStreamReader.EndOfStream) + { + _log.WriteLine(line); + + try + { + var packageId = line.Split(new[] { ". " }, StringSplitOptions.None).Last().Trim(); + + IStrike strike = new JsonStrike(catalogClient, registrationUpdater, packageId); + + line = await indexStreamReader.ReadLineAsync(); + while (!string.IsNullOrEmpty(line) && !line.Contains("Element@")) + { + var url = line.TrimEnd(); + await strike.ProcessCatalogLeafUrlAsync(url); + + // Read next line + line = await indexStreamReader.ReadLineAsync(); + } + + await strike.FinishAsync(); + + // Update cursor file so next time we have less work to do + batchStart++; + await UpdateCursorFileAsync(_cursorFile, batchStart, batchEnd); + } + catch (Exception) + { + UpdateCursorFileAsync(_cursorFile, batchStart, batchEnd).Wait(); + throw; + } + } + } + + await UpdateCursorFileAsync("DONE" + _cursorFile, batchStart, batchEnd); + _log.WriteLine("Finished lightning strike for {0}.", _cursorFile); + } + + private static Task UpdateCursorFileAsync(string cursorFileName, int startIndex, int endIndex) + { + using (var streamWriter = new StreamWriter(cursorFileName)) + { + streamWriter.Write(startIndex); + streamWriter.Write(","); + streamWriter.Write(endIndex); + } + + return Task.FromResult(true); + } + + private IContainer GetAutofacContainer() + { + var services = new ServiceCollection(); + services.AddSingleton(LoggerFactory); + services.AddLogging(); + services.Add(ServiceDescriptor.Scoped(typeof(IOptionsSnapshot<>), typeof(NonCachingOptionsSnapshot<>))); + services.AddSingleton(TelemetryClient); + + services.Configure(config => + { + config.LegacyBaseUrl = _arguments.GetOrDefault(Arguments.StorageBaseAddress); + config.LegacyStorageContainer = _arguments.GetOrDefault(Arguments.StorageContainer); + config.StorageConnectionString = GetConnectionString( + config.StorageConnectionString, + Arguments.StorageAccountName, + Arguments.StorageKeyValue, + Arguments.StorageSuffix); + + config.GzippedBaseUrl = _arguments.GetOrDefault(Arguments.CompressedStorageBaseAddress); + config.GzippedStorageContainer = _arguments.GetOrDefault(Arguments.CompressedStorageContainer); + config.StorageConnectionString = GetConnectionString( + config.StorageConnectionString, + Arguments.CompressedStorageAccountName, + Arguments.CompressedStorageKeyValue, + Arguments.StorageSuffix); + + config.SemVer2BaseUrl = _arguments.GetOrDefault(Arguments.SemVer2StorageBaseAddress); + config.SemVer2StorageContainer = _arguments.GetOrDefault(Arguments.SemVer2StorageContainer); + config.StorageConnectionString = GetConnectionString( + config.StorageConnectionString, + Arguments.SemVer2StorageAccountName, + Arguments.SemVer2StorageKeyValue, + Arguments.StorageSuffix); + + config.GalleryBaseUrl = _arguments.GetOrThrow(Arguments.GalleryBaseAddress); + var contentBaseAddress = _arguments.GetOrThrow(Arguments.ContentBaseAddress); + var flatContainerName = _arguments.GetOrThrow(Arguments.FlatContainerName); + config.FlatContainerBaseUrl = contentBaseAddress.TrimEnd('/') + '/' + flatContainerName; + + config.EnsureSingleSnapshot = true; + }); + + services.AddCatalog2Registration(GlobalTelemetryDimensions); + + var containerBuilder = new ContainerBuilder(); + containerBuilder.AddCatalog2Registration(); + containerBuilder.Populate(services); + + return containerBuilder.Build(); + } + + private string GetConnectionString( + string currentConnectionString, + string accountNameArgument, + string accountKeyArgument, + string endpointSuffixArgument) + { + var builder = new StringBuilder(); + builder.Append("DefaultEndpointsProtocol=https;"); + builder.AppendFormat("AccountName={0};", _arguments.GetOrThrow(accountNameArgument)); + builder.AppendFormat("AccountKey={0};", _arguments.GetOrThrow(accountKeyArgument)); + builder.AppendFormat("EndpointSuffix={0}", _arguments.GetOrDefault(endpointSuffixArgument, "core.windows.net")); + + var connectionString = builder.ToString(); + if (currentConnectionString != null && currentConnectionString != connectionString) + { + throw new InvalidOperationException("The same connection string must be used for all hives."); + } + + return connectionString; + } + + private class JsonStrike : IStrike + { + private readonly ICatalogClient _catalogClient; + private readonly IRegistrationUpdater _updater; + private readonly string _packageId; + private readonly List _urls; + + public JsonStrike( + ICatalogClient catalogClient, + IRegistrationUpdater updater, + string packageId) + { + _catalogClient = catalogClient; + _updater = updater; + _packageId = packageId; + _urls = new List(); + } + + public async Task ProcessCatalogLeafUrlAsync(string url) + { + _urls.Add(url); + if (_urls.Count >= 127) + { + await FinishAsync(); + } + } + + public async Task FinishAsync() + { + if (!_urls.Any()) + { + return; + } + + // Download all of the leaves. + var urlsToDownload = new ConcurrentBag(_urls); + var leaves = new ConcurrentBag(); + await ParallelAsync.Repeat( + async () => + { + await Task.Yield(); + while (urlsToDownload.TryTake(out var url)) + { + var leaf = await _catalogClient.GetLeafAsync(url); + leaves.Add(leaf); + } + }); + + // Build the input to hive updaters. + var entries = new List(); + var entryToLeaf = new Dictionary( + ReferenceEqualityComparer.Default); + foreach (var leaf in leaves) + { + if (leaf.IsPackageDelete() == leaf.IsPackageDetails()) + { + throw new InvalidOperationException("A catalog leaf must be either a package delete or a package details leaf."); + } + + var typeUri = leaf.IsPackageDetails() ? Schema.DataTypes.PackageDetails : Schema.DataTypes.PackageDelete; + + var catalogCommitItem = new CatalogCommitItem( + new Uri(leaf.Url), + leaf.CommitId, + leaf.CommitTimestamp.UtcDateTime, + types: null, + typeUris: new[] { typeUri }, + packageIdentity: new PackageIdentity(_packageId, leaf.ParsePackageVersion())); + + entries.Add(catalogCommitItem); + + if (leaf.IsPackageDetails()) + { + entryToLeaf.Add(catalogCommitItem, (PackageDetailsCatalogLeaf)leaf); + } + } + + // Update the hives. + await _updater.UpdateAsync(_packageId, entries, entryToLeaf); + + _urls.Clear(); + } + } + + private interface IStrike + { + Task ProcessCatalogLeafUrlAsync(string url); + Task FinishAsync(); + } + } +} diff --git a/src/Ng/Jobs/LoopingNgJob.cs b/src/Ng/Jobs/LoopingNgJob.cs new file mode 100644 index 000000000..477d5cf6a --- /dev/null +++ b/src/Ng/Jobs/LoopingNgJob.cs @@ -0,0 +1,60 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.Configuration; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog; + +namespace Ng.Jobs +{ + public abstract class LoopingNgJob : NgJob + { + protected LoopingNgJob( + ILoggerFactory loggerFactory, + ITelemetryClient telemetryClient, + IDictionary telemetryGlobalDimensions) + : base(loggerFactory, telemetryClient, telemetryGlobalDimensions) + { + } + + public override async Task RunAsync(IDictionary arguments, CancellationToken cancellationToken) + { + var intervalSec = arguments.GetOrDefault(Arguments.Interval, Arguments.DefaultInterval); + Logger.LogInformation("Looping job at interval {Interval} seconds.", intervalSec); + + // It can be expensive to initialize, so don't initialize on every run. + // Remember the last time we initialized, and only reinitialize if a specified interval has passed since then. + DateTime? timeLastInitialized = null; + do + { + var timeMustReinitialize = DateTime.UtcNow.Subtract(new TimeSpan(0, 0, 0, + arguments.GetOrDefault(Arguments.ReinitializeIntervalSec, Arguments.DefaultReinitializeIntervalSec))); + + if (!timeLastInitialized.HasValue || timeLastInitialized.Value <= timeMustReinitialize) + { + Logger.LogInformation("Initializing job."); + Init(arguments, cancellationToken); + timeLastInitialized = timeMustReinitialize; + } + + Logger.LogInformation("Running job."); + try + { + await RunInternalAsync(cancellationToken); + Logger.LogInformation("Job finished!"); + } + catch (TransientException te) + { + Logger.LogWarning("Job execution ended due to a transient exception. Exception: {1}, inner exception: {2}.", te, te.InnerException); + } + Logger.LogInformation("Waiting {Interval} seconds before starting job again.", intervalSec); + await Task.Delay(intervalSec * 1000, cancellationToken); + } while (true); + } + } +} diff --git a/src/Ng/Jobs/Monitoring2MonitoringJob.cs b/src/Ng/Jobs/Monitoring2MonitoringJob.cs new file mode 100644 index 000000000..a55c06efa --- /dev/null +++ b/src/Ng/Jobs/Monitoring2MonitoringJob.cs @@ -0,0 +1,84 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.Configuration; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Services.Storage; + +namespace Ng.Jobs +{ + /// + /// Gets s that have and requeue them to be processed by the . + /// + public class Monitoring2MonitoringJob : LoopingNgJob + { + private IPackageMonitoringStatusService _statusService; + private IStorageQueue _queue; + + private const int DefaultMaxQueueSize = 100; + private int _maxRequeueQueueSize; + + public Monitoring2MonitoringJob( + ILoggerFactory loggerFactory, + ITelemetryClient telemetryClient, + IDictionary telemetryGlobalDimensions) + : base(loggerFactory, telemetryClient, telemetryGlobalDimensions) + { + } + + protected override void Init(IDictionary arguments, CancellationToken cancellationToken) + { + var verbose = arguments.GetOrDefault(Arguments.Verbose, false); + _maxRequeueQueueSize = arguments.GetOrDefault(Arguments.MaxRequeueQueueSize, DefaultMaxQueueSize); + + CommandHelpers.AssertAzureStorage(arguments); + + var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); + + _statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory); + + _queue = CommandHelpers.CreateStorageQueue(arguments, PackageValidatorContext.Version); + } + + protected override async Task RunInternalAsync(CancellationToken cancellationToken) + { + var currentMessageCount = await _queue.GetMessageCount(cancellationToken); + if (currentMessageCount > _maxRequeueQueueSize) + { + Logger.LogInformation( + "Can't requeue any invalid packages because the queue has too many messages ({CurrentMessageCount} > {MaxRequeueQueueSize})!", + currentMessageCount, _maxRequeueQueueSize); + return; + } + + var invalidPackages = await _statusService.GetAsync(PackageState.Invalid, cancellationToken); + + await Task.WhenAll(invalidPackages.Select(invalidPackage => + { + try + { + Logger.LogInformation("Requeuing invalid package {PackageId} {PackageVersion}.", + invalidPackage.Package.Id, invalidPackage.Package.Version); + + return _queue.AddAsync( + new PackageValidatorContext(invalidPackage), + cancellationToken); + } + catch (Exception e) + { + Logger.LogError("Failed to requeue invalid package {PackageId} {PackageVersion}: {Exception}", + invalidPackage.Package.Id, invalidPackage.Package.Version, e); + + return Task.FromResult(0); + } + })); + } + } +} \ No newline at end of file diff --git a/src/Ng/Jobs/MonitoringProcessorJob.cs b/src/Ng/Jobs/MonitoringProcessorJob.cs new file mode 100644 index 000000000..f9420e082 --- /dev/null +++ b/src/Ng/Jobs/MonitoringProcessorJob.cs @@ -0,0 +1,376 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Queue.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Services.Configuration; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Services.Sql; +using NuGet.Services.Storage; +using NuGet.Versioning; + +namespace Ng.Jobs +{ + /// + /// Runs validations on packages from a using a . + /// + public class MonitoringProcessorJob : LoopingNgJob + { + private const int DefaultQueueLoopDurationHours = 24; + private const int DefaultQueueDelaySeconds = 30; + private const int DefaultWorkerCount = 8; + private static readonly TimeSpan MaxShutdownTime = TimeSpan.FromMinutes(1); + + private PackageValidator _packageValidator; + private IStorageQueue _queue; + private IPackageMonitoringStatusService _statusService; + private IMonitoringNotificationService _notificationService; + private CollectorHttpClient _client; + private TimeSpan _queueLoopDuration; + private TimeSpan _queueDelay; + private int _workerCount; + + public MonitoringProcessorJob( + ILoggerFactory loggerFactory, + ITelemetryClient telemetryClient, + IDictionary telemetryGlobalDimensions) + : base(loggerFactory, telemetryClient, telemetryGlobalDimensions) + { + } + + protected override void Init(IDictionary arguments, CancellationToken cancellationToken) + { + var gallery = arguments.GetOrThrow(Arguments.Gallery); + var index = arguments.GetOrThrow(Arguments.Index); + var packageBaseAddress = arguments.GetOrThrow(Arguments.ContentBaseAddress); + var source = arguments.GetOrThrow(Arguments.Source); + var requireRepositorySignature = arguments.GetOrDefault(Arguments.RequireRepositorySignature, false); + var verbose = arguments.GetOrDefault(Arguments.Verbose, false); + + var timeoutInSeconds = arguments.GetOrDefault(Arguments.SqlCommandTimeoutInSeconds, 300); + var sqlTimeout = TimeSpan.FromSeconds(timeoutInSeconds); + + var connectionString = arguments.GetOrThrow(Arguments.ConnectionString); + var galleryDbConnection = new AzureSqlConnectionFactory( + connectionString, + SecretInjector, + LoggerFactory.CreateLogger()); + var packageContentUriBuilder = new PackageContentUriBuilder( + arguments.GetOrThrow(Arguments.PackageContentUrlFormat)); + var galleryDatabase = new GalleryDatabaseQueryService( + galleryDbConnection, + packageContentUriBuilder, + TelemetryService, + timeoutInSeconds); + + CommandHelpers.AssertAzureStorage(arguments); + + var monitoringStorageFactory = CommandHelpers.CreateStorageFactory(arguments, verbose); + var auditingStorageFactory = CommandHelpers.CreateSuffixedStorageFactory("Auditing", arguments, verbose); + + var endpointConfiguration = CommandHelpers.GetEndpointConfiguration(arguments); + var messageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose); + + Logger.LogInformation( + "CONFIG gallery: {Gallery} index: {Index} storage: {Storage} auditingStorage: {AuditingStorage} registration cursor uri: {RegistrationCursorUri} flat-container cursor uri: {FlatContainerCursorUri}", + gallery, index, monitoringStorageFactory, auditingStorageFactory, endpointConfiguration.RegistrationCursorUri, endpointConfiguration.FlatContainerCursorUri); + + var validatorConfig = new ValidatorConfiguration( + packageBaseAddress, + requireRepositorySignature); + + _packageValidator = ValidationFactory.CreatePackageValidator( + gallery, + index, + auditingStorageFactory, + validatorConfig, + endpointConfiguration, + messageHandlerFactory, + galleryDatabase, + LoggerFactory); + + _queue = CommandHelpers.CreateStorageQueue(arguments, PackageValidatorContext.Version); + + _statusService = CommandHelpers.GetPackageMonitoringStatusService(arguments, monitoringStorageFactory, LoggerFactory); + + _notificationService = new LoggerMonitoringNotificationService(LoggerFactory.CreateLogger()); + + _client = new CollectorHttpClient(messageHandlerFactory()); + + _queueLoopDuration = TimeSpan.FromHours( + arguments.GetOrDefault( + Arguments.QueueLoopDurationHours, + DefaultQueueLoopDurationHours)); + + _queueDelay = TimeSpan.FromSeconds( + arguments.GetOrDefault( + Arguments.QueueDelaySeconds, + DefaultQueueDelaySeconds)); + + _workerCount = arguments.GetOrDefault(Arguments.WorkerCount, DefaultWorkerCount); + + SetUserAgentString(); + } + + /// + /// Unfortunately, we have to use reflection to set the user agent string as we'd like to. + /// See https://github.com/NuGet/Home/issues/7464 + /// + private static void SetUserAgentString() + { + var userAgentString = UserAgentUtility.GetUserAgent(); + + typeof(UserAgent) + .GetProperty(nameof(UserAgent.UserAgentString)) + .SetValue(null, userAgentString); + } + + protected override async Task RunInternalAsync(CancellationToken cancellationToken) + { + // We should stop processing messages if the job runner cancels us. + var queueMessageCancellationToken = cancellationToken; + + // We should stop dequeuing more messages if too much time elapses. + Logger.LogInformation("Processing messages for {Duration} before restarting the job loop.", _queueLoopDuration); + using (var queueLoopCancellationTokenSource = new CancellationTokenSource(_queueLoopDuration)) + using (var timeoutCancellationTokenSource = new CancellationTokenSource()) + { + var queueLoopCancellationToken = queueLoopCancellationTokenSource.Token; + + var workerId = 0; + var allWorkersTask = ParallelAsync.Repeat( + () => ProcessPackagesAsync( + Interlocked.Increment(ref workerId), + queueLoopCancellationToken, + queueMessageCancellationToken), + _workerCount); + + // Wait for a specific amount of time past the loop duration. If a worker task is hanging for whatever + // reason we don't want to the shutdown to be blocked indefinitely. + // + // Imagine one worker is stuck and all of the rest of the workers have successfully stopped consuming + // messages. This would mean that this process is stuck in a seemingly "healthy" state (no exceptions, + // the process is still alive) but it will never terminate and no queue messages will be processed. By + // design all jobs must be resilient to unexpected termination (machine shutdown, etc) so not waiting + // for a slow worker task to gracefully finish is acceptable. + var loopDurationPlusShutdownTask = Task.Delay(_queueLoopDuration.Add(MaxShutdownTime), timeoutCancellationTokenSource.Token); + + var firstTask = await Task.WhenAny(allWorkersTask, loopDurationPlusShutdownTask); + if (firstTask == loopDurationPlusShutdownTask) + { + Logger.LogWarning("Not all workers shut down gracefully after {Duration}.", MaxShutdownTime); + } + else + { + timeoutCancellationTokenSource.Cancel(); + Logger.LogInformation("All workers gracefully shut down."); + } + } + } + + /// The integer identifier of this worker, for diagnostic purposes. + /// + /// When this token is cancelled, the process will stop dequeuing new messages. + /// Messages that we already dequeued will continue being processed. + /// + /// + /// When this token is cancelled, messages that have already been dequeued will stop being processed. + /// The process will also stop dequeuing new messages because they wouldn't be processed anyway. + /// + private async Task ProcessPackagesAsync( + int workerId, + CancellationToken queueLoopCancellationToken, + CancellationToken queueMessageCancellationToken) + { + // We will never listen to cancellation of the queue loop token individually. + // If the queue message token is cancelled, we will always want to stop dequeuing new messages. + // We combine the two tokens here so we don't have to call "IsCancellationRequested" multiple times. + using (var combinedQueueLoopCancellationTokenSource = CancellationTokenSource + .CreateLinkedTokenSource( + queueLoopCancellationToken, + queueMessageCancellationToken)) + using (Logger.BeginScope("Worker {WorkerId} is processing messages.", workerId)) + { + var combinedQueueLoopCancellationToken = combinedQueueLoopCancellationTokenSource.Token; + await HandleQueueMessagesAsync(combinedQueueLoopCancellationToken, queueMessageCancellationToken); + } + } + + private async Task HandleQueueMessagesAsync( + CancellationToken combinedQueueLoopCancellationToken, + CancellationToken queueMessageCancellationToken) + { + Logger.LogInformation("Beginning fetching queue messages."); + do + { + var shouldWaitBeforeNextMessage = false; + try + { + Logger.LogInformation("Fetching next queue message."); + var queueMessage = await _queue.GetNextAsync(queueMessageCancellationToken); + await HandleQueueMessageAsync(queueMessage, queueMessageCancellationToken); + + if (queueMessage == null) + { + Logger.LogInformation( + "Failed to fetch last message or no messages left in queue."); + + shouldWaitBeforeNextMessage = true; + } + } + catch (Exception e) + { + Logger.LogCritical( + NuGet.Services.Metadata.Catalog.Monitoring.LogEvents.QueueMessageFatalFailure, + e, + "Failed to process queue message."); + + shouldWaitBeforeNextMessage = true; + } + + if (shouldWaitBeforeNextMessage && !combinedQueueLoopCancellationToken.IsCancellationRequested) + { + Logger.LogInformation( + "Waiting {QueueDelaySeconds} seconds before polling again.", + _queueDelay.TotalSeconds); + + try + { + await Task.Delay(_queueDelay, combinedQueueLoopCancellationToken); + } + catch (TaskCanceledException) + { + Logger.LogInformation("Stopped waiting before polling because task was cancelled."); + } + } + } while (!combinedQueueLoopCancellationToken.IsCancellationRequested); + Logger.LogInformation("Finished fetching queue messages."); + } + + private async Task HandleQueueMessageAsync( + StorageQueueMessage queueMessage, + CancellationToken token) + { + if (queueMessage == null) + { + return; + } + + var queuedContext = queueMessage.Contents; + var messageWasProcessed = false; + + try + { + await RunPackageValidatorAsync(queuedContext, token); + // The validations ran successfully and were saved to storage. + // We can remove the message from the queue because it was processed. + messageWasProcessed = true; + } + catch (Exception validationFailedToRunException) + { + try + { + // Validations failed to run! Save this failed status to storage. + await SaveFailedPackageMonitoringStatusAsync(queuedContext, validationFailedToRunException, token); + // We can then remove the message from the queue because this failed status can be used to requeue the message. + messageWasProcessed = true; + } + catch (Exception failedValidationSaveFailureException) + { + // We failed to run validations and failed to save the failed validation! + // We were not able to process this message. We need to log the exceptions so we can debug the issue. + throw new AggregateException( + "Validations failed to run and saving unsuccessful validation failed!", + new[] { validationFailedToRunException, failedValidationSaveFailureException }); + } + } + + // If we failed to run validations and failed to save the failed validation, we cannot remove the message from the queue. + if (messageWasProcessed) + { + try + { + await _queue.RemoveAsync(queueMessage, token); + } + catch (StorageException storageException) + { + if (storageException.RequestInformation.ExtendedErrorInformation.ErrorCode == QueueErrorCodeStrings.MessageNotFound) + { + Logger.LogWarning( + NuGet.Services.Metadata.Catalog.Monitoring.LogEvents.QueueMessageRemovalFailure, + storageException, + "Queue message for {PackageId} {PackageVersion} no longer exists. Message was likely already processed.", + queuedContext.Package.Id, queuedContext.Package.Version); + } + else + { + Logger.LogCritical( + NuGet.Services.Metadata.Catalog.Monitoring.LogEvents.QueueMessageRemovalFailure, + storageException, + "Failed to remove queue message."); + } + } + } + } + + private async Task RunPackageValidatorAsync( + PackageValidatorContext queuedContext, + CancellationToken token) + { + var feedPackage = queuedContext.Package; + Logger.LogInformation("Running PackageValidator on PackageValidatorContext for {PackageId} {PackageVersion}.", feedPackage.Id, feedPackage.Version); + var catalogEntries = queuedContext.CatalogEntries; + var existingStatus = await _statusService.GetAsync(feedPackage, token); + if (catalogEntries != null && existingStatus?.ValidationResult?.CatalogEntries != null && CompareCatalogEntries(catalogEntries, existingStatus.ValidationResult.CatalogEntries)) + { + // A newer catalog entry of this package has already been validated. + Logger.LogInformation("A newer catalog entry of {PackageId} {PackageVersion} has already been processed ({OldCommitTimeStamp} < {NewCommitTimeStamp}).", + feedPackage.Id, feedPackage.Version, + catalogEntries.Max(c => c.CommitTimeStamp), + existingStatus.ValidationResult.CatalogEntries.Max(c => c.CommitTimeStamp)); + + return; + } + + var context = new PackageValidatorContext(feedPackage, catalogEntries); + var result = await _packageValidator.ValidateAsync(context, _client, token); + await _notificationService.OnPackageValidationFinishedAsync(result, token); + var status = new PackageMonitoringStatus(result); + PackageMonitoringStatusAccessConditionHelper.UpdateFromExisting(status, existingStatus); + await _statusService.UpdateAsync(status, token); + } + + private async Task SaveFailedPackageMonitoringStatusAsync( + PackageValidatorContext queuedContext, + Exception exception, + CancellationToken token) + { + var queuedVersion = queuedContext.Package.Version; + var version = NuGetVersion.TryParse(queuedVersion, out var parsedVersion) + ? parsedVersion.ToFullString() : queuedVersion; + + var feedPackage = new FeedPackageIdentity(queuedContext.Package.Id, version); + await _notificationService.OnPackageValidationFailedAsync(feedPackage.Id, feedPackage.Version, exception, token); + var status = new PackageMonitoringStatus(feedPackage, exception); + await _statusService.UpdateAsync(status, token); + } + + /// + /// Returns if the newest entry in is older than the newest entry in . + /// + private bool CompareCatalogEntries(IEnumerable first, IEnumerable second) + { + return first.Max(c => c.CommitTimeStamp) < second.Max(c => c.CommitTimeStamp); + } + } +} \ No newline at end of file diff --git a/src/Ng/Jobs/NgJob.cs b/src/Ng/Jobs/NgJob.cs new file mode 100644 index 000000000..993dfab3a --- /dev/null +++ b/src/Ng/Jobs/NgJob.cs @@ -0,0 +1,79 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.KeyVault; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog; + +namespace Ng.Jobs +{ + public abstract class NgJob + { + protected ISecretInjector SecretInjector { get; private set; } + + protected readonly IDictionary GlobalTelemetryDimensions; + protected readonly ITelemetryClient TelemetryClient; + protected readonly ITelemetryService TelemetryService; + protected readonly ILoggerFactory LoggerFactory; + protected readonly ILogger Logger; + + protected int MaxDegreeOfParallelism { get; set; } + + protected NgJob( + ILoggerFactory loggerFactory, + ITelemetryClient telemetryClient, + IDictionary telemetryGlobalDimensions) + { + LoggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + TelemetryClient = telemetryClient ?? throw new ArgumentNullException(nameof(telemetryClient)); + GlobalTelemetryDimensions = telemetryGlobalDimensions ?? throw new ArgumentNullException(nameof(telemetryGlobalDimensions)); + TelemetryService = new TelemetryService(telemetryClient, telemetryGlobalDimensions); + + // We want to make a logger using the subclass of this job. + // GetType returns the subclass of this instance which we can then use to create a logger. + Logger = LoggerFactory.CreateLogger(GetType()); + + // Enable greater HTTP parallelization. + ServicePointManager.DefaultConnectionLimit = 64; + ServicePointManager.MaxServicePointIdleTime = 10000; + + MaxDegreeOfParallelism = ServicePointManager.DefaultConnectionLimit; + } + + public static string GetUsageBase() + { + return "Usage: ng [" + string.Join("|", NgJobFactory.JobMap.Keys) + "] " + + $"[-{Arguments.VaultName} " + + $"-{Arguments.UseManagedIdentity} true|false " + + $"-{Arguments.ClientId} Should not be set if {Arguments.UseManagedIdentity} is true" + + $"-{Arguments.CertificateThumbprint} Should not be set if {Arguments.UseManagedIdentity} is true" + + $"[-{Arguments.ValidateCertificate} true|false]]"; + } + + public virtual string GetUsage() + { + return GetUsageBase(); + } + + public void SetSecretInjector(ISecretInjector secretInjector) + { + SecretInjector = secretInjector; + } + + protected abstract void Init(IDictionary arguments, CancellationToken cancellationToken); + + protected abstract Task RunInternalAsync(CancellationToken cancellationToken); + + public virtual async Task RunAsync(IDictionary arguments, CancellationToken cancellationToken) + { + Init(arguments, cancellationToken); + await RunInternalAsync(cancellationToken); + } + } +} \ No newline at end of file diff --git a/src/Ng/Monitoring2Monitoring.nuspec b/src/Ng/Monitoring2Monitoring.nuspec new file mode 100644 index 000000000..69ba3d09d --- /dev/null +++ b/src/Ng/Monitoring2Monitoring.nuspec @@ -0,0 +1,16 @@ + + + + Monitoring2Monitoring + $version$ + .NET Foundation + .NET Foundation + The Monitoring2Monitoring job. + Copyright .NET Foundation + + + + + + + \ No newline at end of file diff --git a/src/Ng/MonitoringProcessor.nuspec b/src/Ng/MonitoringProcessor.nuspec new file mode 100644 index 000000000..df4188542 --- /dev/null +++ b/src/Ng/MonitoringProcessor.nuspec @@ -0,0 +1,16 @@ + + + + MonitoringProcessor + $version$ + .NET Foundation + .NET Foundation + The MonitoringProcessor job. + Copyright .NET Foundation + + + + + + + \ No newline at end of file diff --git a/src/Ng/Ng.Operations.nuspec b/src/Ng/Ng.Operations.nuspec new file mode 100644 index 000000000..0e9b76cca --- /dev/null +++ b/src/Ng/Ng.Operations.nuspec @@ -0,0 +1,15 @@ + + + + Ng.Operations + $version$ + Ng.Operations + .NET Foundation + .NET Foundation + Ng package, but without the service scripts. Intended for ops tools that are manually ran. + Copyright .NET Foundation + + + + + \ No newline at end of file diff --git a/src/Ng/Ng.csproj b/src/Ng/Ng.csproj new file mode 100644 index 000000000..2460063ce --- /dev/null +++ b/src/Ng/Ng.csproj @@ -0,0 +1,184 @@ + + + + + + Debug + AnyCPU + {5234D86F-2C0E-4181-AAB7-BBDA3253B4E1} + Exe + Properties + Ng + Ng + v4.7.2 + 512 + + + + true + win + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + false + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Designer + + + + + + + + Designer + + + + + + {e97f23b8-ecb0-4afa-b00c-015c39395fef} + NuGet.Services.Metadata.Catalog + + + {5ABE8807-2209-4948-9FC5-1980A507C47A} + NuGet.Jobs.Catalog2Registration + + + {d44c2e89-2d98-44bd-8712-8ccbe4e67c9c} + NuGet.Protocol.Catalog + + + {1745a383-d0be-484b-81eb-27b20f6ac6c5} + NuGet.Services.Metadata.Catalog.Monitoring + + + {C3F9A738-9759-4B2B-A50D-6507B28A659B} + NuGet.Services.V3 + + + + + False + Microsoft .NET Framework 4.5 %28x86 and x64%29 + true + + + False + .NET Framework 3.5 SP1 Client Profile + false + + + False + .NET Framework 3.5 SP1 + false + + + + + PreserveNewest + + + + + 0.3.0 + runtime; build; native; contentfiles; analyzers + all + + + 2.2.0 + + + 3.1.0 + + + 4.8.0 + runtime; build; native; contentfiles; analyzers + all + + + 2.75.0 + + + 2.75.0 + + + 2.75.0 + + + 2.75.0 + + + 4.0.0 + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + + + \ No newline at end of file diff --git a/src/Ng/Ng.csproj.DotSettings b/src/Ng/Ng.csproj.DotSettings new file mode 100644 index 000000000..73e96563f --- /dev/null +++ b/src/Ng/Ng.csproj.DotSettings @@ -0,0 +1,2 @@ + + CSharp60 \ No newline at end of file diff --git a/src/Ng/NgJobFactory.cs b/src/Ng/NgJobFactory.cs new file mode 100644 index 000000000..8daef6ab2 --- /dev/null +++ b/src/Ng/NgJobFactory.cs @@ -0,0 +1,44 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using Ng.Jobs; +using NuGet.Services.Logging; + +namespace Ng +{ + public static class NgJobFactory + { + public static IDictionary JobMap = new Dictionary() + { + { "db2catalog", typeof(Db2CatalogJob) }, + { "db2monitoring", typeof(Db2MonitoringJob) }, + { "catalog2dnx", typeof(Catalog2DnxJob) }, + { "lightning", typeof(LightningJob) }, + { "catalog2monitoring", typeof(Catalog2MonitoringJob) }, + { "monitoring2monitoring", typeof(Monitoring2MonitoringJob) }, + { "monitoringprocessor", typeof(MonitoringProcessorJob) }, + { "catalog2packagefixup", typeof(Catalog2PackageFixupJob) }, + { "catalog2icon", typeof(Catalog2IconJob) }, + }; + + public static NgJob GetJob( + string jobName, + ILoggerFactory loggerFactory, + ITelemetryClient telemetryClient, + IDictionary telemetryGlobalDimensions) + { + if (JobMap.ContainsKey(jobName)) + { + return + (NgJob) + JobMap[jobName].GetConstructor(new[] { typeof(ILoggerFactory), typeof(ITelemetryClient), typeof(IDictionary) }) + .Invoke(new object[] { loggerFactory, telemetryClient, telemetryGlobalDimensions }); + } + + throw new ArgumentException("Missing or invalid job name!"); + } + } +} diff --git a/src/Ng/Program.cs b/src/Ng/Program.cs new file mode 100644 index 000000000..e18a61a14 --- /dev/null +++ b/src/Ng/Program.cs @@ -0,0 +1,212 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Autofac; +using Autofac.Extensions.DependencyInjection; +using Microsoft.ApplicationInsights; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Ng.Jobs; +using NuGet.Services.Configuration; +using NuGet.Services.KeyVault; +using NuGet.Services.Logging; +using Serilog; +using Serilog.Events; + +namespace Ng +{ + public class Program + { + private const string HeartbeatProperty_JobLoopExitCode = "JobLoopExitCode"; + + private static Microsoft.Extensions.Logging.ILogger _logger; + + public static void Main(string[] args) + { + MainAsync(args).GetAwaiter().GetResult(); + } + + public static async Task MainAsync(string[] args) + { + if (args.Length > 0 && string.Equals("dbg", args[0], StringComparison.OrdinalIgnoreCase)) + { + args = args.Skip(1).ToArray(); + Debugger.Launch(); + } + + NgJob job = null; + ApplicationInsightsConfiguration applicationInsightsConfiguration = null; + int exitCode = 0; + + try + { + // Get arguments + var arguments = CommandHelpers.GetArguments(args, 1, out var secretInjector); + + // Ensure that SSLv3 is disabled and that Tls v1.2 is enabled. + ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3; + ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; + + // Determine the job name + if (args.Length == 0) + { + throw new ArgumentException("Missing job name argument."); + } + + var jobName = args[0]; + var instanceName = arguments.GetOrDefault(Arguments.InstanceName, jobName); + var instrumentationKey = arguments.GetOrDefault(Arguments.InstrumentationKey); + var heartbeatIntervalSeconds = arguments.GetOrDefault(Arguments.HeartbeatIntervalSeconds); + + applicationInsightsConfiguration = ConfigureApplicationInsights( + instrumentationKey, + heartbeatIntervalSeconds, + jobName, + instanceName, + out var telemetryClient, + out var telemetryGlobalDimensions); + + var loggerFactory = ConfigureLoggerFactory(applicationInsightsConfiguration); + + InitializeServiceProvider( + arguments, + secretInjector, + applicationInsightsConfiguration, + telemetryClient, + loggerFactory); + + job = NgJobFactory.GetJob(jobName, loggerFactory, telemetryClient, telemetryGlobalDimensions); + job.SetSecretInjector(secretInjector); + + // This tells Application Insights that, even though a heartbeat is reported, + // the state of the application is unhealthy when the exitcode is different from zero. + // The heartbeat metadata is enriched with the job loop exit code. + applicationInsightsConfiguration.DiagnosticsTelemetryModule?.AddOrSetHeartbeatProperty( + HeartbeatProperty_JobLoopExitCode, + exitCode.ToString(), + isHealthy: exitCode == 0); + + var cancellationTokenSource = new CancellationTokenSource(); + await job.RunAsync(arguments, cancellationTokenSource.Token); + exitCode = 0; + } + catch (ArgumentException ae) + { + exitCode = 1; + _logger?.LogError("A required argument was not found or was malformed/invalid: {Exception}", ae); + + Console.WriteLine(job != null ? job.GetUsage() : NgJob.GetUsageBase()); + } + catch (KeyNotFoundException knfe) + { + exitCode = 1; + _logger?.LogError("An expected key was not found. One possible cause of this is required argument has not been provided: {Exception}", knfe); + + Console.WriteLine(job != null ? job.GetUsage() : NgJob.GetUsageBase()); + } + catch (Exception e) + { + exitCode = 1; + _logger?.LogCritical("A critical exception occured in ng.exe! {Exception}", e); + } + + applicationInsightsConfiguration.DiagnosticsTelemetryModule?.SetHeartbeatProperty( + HeartbeatProperty_JobLoopExitCode, + exitCode.ToString(), + isHealthy: exitCode == 0); + + Trace.Close(); + applicationInsightsConfiguration?.TelemetryConfiguration.TelemetryChannel.Flush(); + } + + /// + /// This mimics the approach taking in NuGet.Job's JsonConfigurationJob. We add common infrastructure to the + /// dependency injection container and load Autofac modules in the current assembly. This gives us a hook to + /// customize the initialization of this program. Today this service provider is only used for modifying + /// telemetry configuration but it could eventually be used in the . + /// + private static IServiceProvider InitializeServiceProvider( + IDictionary arguments, + ISecretInjector secretInjector, + ApplicationInsightsConfiguration applicationInsightsConfiguration, + ITelemetryClient telemetryClient, + ILoggerFactory loggerFactory) + { + var configurationBuilder = new ConfigurationBuilder() + .AddInMemoryCollection(arguments); + + IServiceCollection services = new ServiceCollection(); + services.AddSingleton(secretInjector); + services.AddSingleton(applicationInsightsConfiguration.TelemetryConfiguration); + services.AddSingleton(configurationBuilder.Build()); + + services.Add(ServiceDescriptor.Scoped(typeof(IOptionsSnapshot<>), typeof(NonCachingOptionsSnapshot<>))); + services.AddSingleton(loggerFactory); + services.AddLogging(); + + services.AddSingleton(telemetryClient); + + var containerBuilder = new ContainerBuilder(); + containerBuilder.Populate(services); + containerBuilder.RegisterAssemblyModules(typeof(Program).Assembly); + + return new AutofacServiceProvider(containerBuilder.Build()); + } + + private static ILoggerFactory ConfigureLoggerFactory(ApplicationInsightsConfiguration applicationInsightsConfiguration) + { + var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: true); + loggerConfiguration.WriteTo.File("Log.txt", retainedFileCountLimit: 3, fileSizeLimitBytes: 1000000, rollOnFileSizeLimit: true); + + var loggerFactory = LoggingSetup.CreateLoggerFactory( + loggerConfiguration, + LogEventLevel.Debug, + applicationInsightsConfiguration.TelemetryConfiguration); + + // Create a logger that is scoped to this class (only) + _logger = loggerFactory.CreateLogger(); + return loggerFactory; + } + + private static ApplicationInsightsConfiguration ConfigureApplicationInsights( + string instrumentationKey, + int heartbeatIntervalSeconds, + string jobName, + string instanceName, + out ITelemetryClient telemetryClient, + out IDictionary telemetryGlobalDimensions) + { + ApplicationInsightsConfiguration applicationInsightsConfiguration; + if (heartbeatIntervalSeconds == 0) + { + applicationInsightsConfiguration = ApplicationInsights.Initialize(instrumentationKey); + } + else + { + applicationInsightsConfiguration = ApplicationInsights.Initialize( + instrumentationKey, + TimeSpan.FromSeconds(heartbeatIntervalSeconds)); + } + + telemetryClient = new TelemetryClientWrapper( + new TelemetryClient(applicationInsightsConfiguration.TelemetryConfiguration)); + + telemetryGlobalDimensions = new Dictionary(); + + // Enrich telemetry with job properties and global custom dimensions + applicationInsightsConfiguration.TelemetryConfiguration.TelemetryInitializers.Add( + new JobPropertiesTelemetryInitializer(jobName, instanceName, telemetryGlobalDimensions)); + + return applicationInsightsConfiguration; + } + } +} diff --git a/src/Ng/Properties/AssemblyInfo.cs b/src/Ng/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..5201c6a76 --- /dev/null +++ b/src/Ng/Properties/AssemblyInfo.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("Ng")] +[assembly: AssemblyDescription("Ng")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany(".NET Foundation")] +[assembly: AssemblyProduct("NuGet")] +[assembly: AssemblyCopyright("\x00a9 .NET Foundation. All rights reserved.")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] +[assembly: ComVisible(false)] +[assembly: Guid("e70accdd-c85c-4f55-868b-20d2ffb1293b")] diff --git a/src/Ng/Scripts/Functions.ps1 b/src/Ng/Scripts/Functions.ps1 new file mode 100644 index 000000000..a8bff40fc --- /dev/null +++ b/src/Ng/Scripts/Functions.ps1 @@ -0,0 +1,30 @@ +Function Uninstall-NuGetService() { + Param ([string]$ServiceName) + + if (Get-Service $ServiceName -ErrorAction SilentlyContinue) + { + Write-Host Removing service $ServiceName... + Stop-Service $ServiceName -Force + sc.exe delete $ServiceName + Write-Host Removed service $ServiceName. + } else { + Write-Host Skipping removal of service $ServiceName - no such service exists. + } +} + +Function Install-NuGetService() { + Param ([string]$ServiceName, [string]$ServiceTitle, [string]$ScriptToRun) + + Write-Host Installing service $ServiceName... + + $installService = "nssm install $ServiceName $ScriptToRun" + cmd /C $installService + + Set-Service -Name $ServiceName -DisplayName "$ServiceTitle - $ServiceName" -Description "Runs $ServiceTitle." -StartupType Automatic + sc.exe failure $ServiceName reset= 30 actions= restart/5000 + + # Run service + net start $ServiceName + + Write-Host Installed service $ServiceName. +} \ No newline at end of file diff --git a/src/Ng/Scripts/PostDeploy.ps1 b/src/Ng/Scripts/PostDeploy.ps1 new file mode 100644 index 000000000..7d5183d5b --- /dev/null +++ b/src/Ng/Scripts/PostDeploy.ps1 @@ -0,0 +1,18 @@ +. .\Functions.ps1 + +$jobsToInstall = $OctopusParameters["Jobs.ServiceNames"].Split("{,}") + +Write-Host Installing services... + +$currentDirectory = [string](Get-Location) + +$jobsToInstall.Split("{;}") | %{ + $serviceName = $_ + $serviceTitle = $OctopusParameters["Jobs.$serviceName.Title"] + $scriptToRun = $OctopusParameters["Jobs.$serviceName.Script"] + $scriptToRun = "$currentDirectory\$scriptToRun" + + Install-NuGetService -ServiceName $serviceName -ServiceTitle $serviceTitle -ScriptToRun $scriptToRun +} + +Write-Host Installed services. \ No newline at end of file diff --git a/src/Ng/Scripts/PreDeploy.ps1 b/src/Ng/Scripts/PreDeploy.ps1 new file mode 100644 index 000000000..ef711a912 --- /dev/null +++ b/src/Ng/Scripts/PreDeploy.ps1 @@ -0,0 +1,11 @@ +. .\Functions.ps1 + +$jobsToInstall = $OctopusParameters["Jobs.ServiceNames"].Split("{,}") + +Write-Host Removing services... + +$jobsToInstall.Split("{;}") | %{ + Uninstall-NuGetService -ServiceName $_ +} + +Write-Host Removed services. \ No newline at end of file diff --git a/src/Ng/Scripts/nssm.exe b/src/Ng/Scripts/nssm.exe new file mode 100644 index 000000000..6ccfe3cfb Binary files /dev/null and b/src/Ng/Scripts/nssm.exe differ diff --git a/src/Ng/StorageAccessHandler.cs b/src/Ng/StorageAccessHandler.cs new file mode 100644 index 000000000..ee40fdda4 --- /dev/null +++ b/src/Ng/StorageAccessHandler.cs @@ -0,0 +1,35 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace Ng +{ + public class StorageAccessHandler : DelegatingHandler + { + private readonly string _catalogBaseAddress; + private readonly string _storageBaseAddress; + + public StorageAccessHandler(string catalogBaseAddress, string storageBaseAddress, HttpMessageHandler handler) + : base(handler) + { + _catalogBaseAddress = catalogBaseAddress; + _storageBaseAddress = storageBaseAddress; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + string requestUri = request.RequestUri.AbsoluteUri; + + if (requestUri.StartsWith(_catalogBaseAddress)) + { + string newRequestUri = _storageBaseAddress + requestUri.Substring(_catalogBaseAddress.Length); + request.RequestUri = new Uri(newRequestUri); + } + + return base.SendAsync(request, cancellationToken); + } + } +} diff --git a/src/Ng/UserAgentUtility.cs b/src/Ng/UserAgentUtility.cs new file mode 100644 index 000000000..2e96d877b --- /dev/null +++ b/src/Ng/UserAgentUtility.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; + +namespace Ng +{ + public class UserAgentUtility + { + /// + /// Returns a user agent built using the entry assembly's name and version. + /// + public static string GetUserAgent() + { + var assembly = Assembly.GetEntryAssembly(); + var assemblyName = assembly.GetName().Name; + var assemblyVersion = assembly.GetCustomAttribute()?.InformationalVersion ?? "0.0.0"; + + return $"{assemblyName}/{assemblyVersion}"; + } + } +} diff --git a/src/Ng/lightning-template.txt b/src/Ng/lightning-template.txt new file mode 100644 index 000000000..ce9c7b990 --- /dev/null +++ b/src/Ng/lightning-template.txt @@ -0,0 +1,35 @@ +@Echo off + +:Top + +title [cursorFile] + +@echo starting + +start /w ng.exe lightning ^ + -command strike ^ + -indexFile "[indexFile]" ^ + -cursorFile "[cursorFile]" ^ + -contentBaseAddress "[contentBaseAddress]" ^ + -galleryBaseAddress "[galleryBaseAddress]" ^ + -storageBaseAddress "[storageBaseAddress]" ^ + -storageAccountName "[storageAccountName]" ^ + -storageKeyValue "[storageKeyValue]" ^ + -storageContainer "[storageContainer]" ^ + -compressedStorageBaseAddress "[compressedStorageBaseAddress]" ^ + -compressedStorageAccountName "[compressedStorageAccountName]" ^ + -compressedStorageKeyValue "[compressedStorageKeyValue]" ^ + -compressedStorageContainer "[compressedStorageContainer]" ^ + -semVer2StorageBaseAddress "[semVer2StorageBaseAddress]" ^ + -semVer2StorageAccountName "[semVer2StorageAccountName]" ^ + -semVer2StorageKeyValue "[semVer2StorageKeyValue]" ^ + -semVer2StorageContainer "[semVer2StorageContainer]" ^ + [optionalArguments] + +@echo finished + +If exist DONE[cursorFile] GOTO EOF + +Goto Top + +:EOF \ No newline at end of file diff --git a/src/NuGet.ApplicationInsights.Owin/ApplicationInsights.config b/src/NuGet.ApplicationInsights.Owin/ApplicationInsights.config new file mode 100644 index 000000000..75e12df9d --- /dev/null +++ b/src/NuGet.ApplicationInsights.Owin/ApplicationInsights.config @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/src/NuGet.ApplicationInsights.Owin/ExceptionTrackingMiddleware.cs b/src/NuGet.ApplicationInsights.Owin/ExceptionTrackingMiddleware.cs new file mode 100644 index 000000000..b4c74e51e --- /dev/null +++ b/src/NuGet.ApplicationInsights.Owin/ExceptionTrackingMiddleware.cs @@ -0,0 +1,42 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.Owin; + +namespace NuGet.ApplicationInsights.Owin +{ + public class ExceptionTrackingMiddleware + : OwinMiddleware + { + private readonly TelemetryClient _telemetryClient; + + public ExceptionTrackingMiddleware(OwinMiddleware next) + : this(next, null) + { + } + + public ExceptionTrackingMiddleware(OwinMiddleware next, TelemetryConfiguration telemetryConfiguration) + : base(next) + { + _telemetryClient = telemetryConfiguration == null + ? new TelemetryClient() + : new TelemetryClient(telemetryConfiguration); + } + + public override async Task Invoke(IOwinContext context) + { + try + { + await this.Next.Invoke(context); + } + catch (Exception e) + { + this._telemetryClient.TrackException(e); + } + } + } +} diff --git a/src/NuGet.ApplicationInsights.Owin/NuGet.ApplicationInsights.Owin.csproj b/src/NuGet.ApplicationInsights.Owin/NuGet.ApplicationInsights.Owin.csproj new file mode 100644 index 000000000..a10b489bf --- /dev/null +++ b/src/NuGet.ApplicationInsights.Owin/NuGet.ApplicationInsights.Owin.csproj @@ -0,0 +1,102 @@ + + + + + + Debug + AnyCPU + + + 2.0 + {717E9A81-75C5-418E-92ED-18CAC55BC345} + Library + Properties + NuGet.ApplicationInsights.Owin + NuGet.ApplicationInsights.Owin + v4.7.2 + + + + true + win + + + .NET Foundation + https://github.com/NuGet/NuGet.Services.Metadata/blob/master/LICENSE + https://github.com/NuGet/NuGet.Services.Metadata + OWIN Middleware that provides context to ApplicationInsights telemetry. + nuget;application;insights;owin + Copyright .NET Foundation + + + true + full + false + bin\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\ + TRACE + prompt + 4 + + + + + False + + + + + + + + + + + + + + + + + + + + 0.3.0 + runtime; build; native; contentfiles; analyzers + all + + + 2.12.0 + + + 3.0.1 + + + 4.8.0 + runtime; build; native; contentfiles; analyzers + all + + + + 10.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + + + \ No newline at end of file diff --git a/src/NuGet.ApplicationInsights.Owin/OwinRequestIdContext.cs b/src/NuGet.ApplicationInsights.Owin/OwinRequestIdContext.cs new file mode 100644 index 000000000..5be1492ae --- /dev/null +++ b/src/NuGet.ApplicationInsights.Owin/OwinRequestIdContext.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Runtime.Remoting.Messaging; + +namespace NuGet.ApplicationInsights.Owin +{ + public static class OwinRequestIdContext + { + public static void Set(string value) + { + if (!string.IsNullOrEmpty(value)) + { + CallContext.LogicalSetData(RequestTrackingMiddleware.OwinRequestIdKey, value); + } + } + + public static string Get() + { + return CallContext.LogicalGetData(RequestTrackingMiddleware.OwinRequestIdKey) as string; + } + + public static void Clear() + { + CallContext.LogicalSetData(RequestTrackingMiddleware.OwinRequestIdKey, null); + } + } +} \ No newline at end of file diff --git a/src/NuGet.ApplicationInsights.Owin/OwinRequestIdTelemetryInitializer.cs b/src/NuGet.ApplicationInsights.Owin/OwinRequestIdTelemetryInitializer.cs new file mode 100644 index 000000000..f8354107e --- /dev/null +++ b/src/NuGet.ApplicationInsights.Owin/OwinRequestIdTelemetryInitializer.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.Extensibility; + +namespace NuGet.ApplicationInsights.Owin +{ + public class OwinRequestIdTelemetryInitializer + : ITelemetryInitializer + { + public void Initialize(ITelemetry telemetry) + { + var owinRequestId = OwinRequestIdContext.Get(); + if (owinRequestId != null) + { + telemetry.Context.Operation.Id = owinRequestId; + } + } + } +} \ No newline at end of file diff --git a/src/NuGet.ApplicationInsights.Owin/Properties/AssemblyInfo.cs b/src/NuGet.ApplicationInsights.Owin/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..cc24b03c8 --- /dev/null +++ b/src/NuGet.ApplicationInsights.Owin/Properties/AssemblyInfo.cs @@ -0,0 +1,24 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("NuGet.ApplicationInsights.Owin")] +[assembly: AssemblyDescription("NuGet.ApplicationInsights.Owin")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("NuGet.ApplicationInsights.Owin")] +[assembly: AssemblyCopyright("Copyright © 2016")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("717e9a81-75c5-418e-92ed-18cac55bc345")] \ No newline at end of file diff --git a/src/NuGet.ApplicationInsights.Owin/RequestTrackingMiddleware.cs b/src/NuGet.ApplicationInsights.Owin/RequestTrackingMiddleware.cs new file mode 100644 index 000000000..8e829fffb --- /dev/null +++ b/src/NuGet.ApplicationInsights.Owin/RequestTrackingMiddleware.cs @@ -0,0 +1,97 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Threading.Tasks; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.Owin; + +namespace NuGet.ApplicationInsights.Owin +{ + public class RequestTrackingMiddleware + : OwinMiddleware + { + public const string OwinRequestIdKey = "owin.RequestId"; + + private readonly TelemetryClient _telemetryClient; + + public RequestTrackingMiddleware(OwinMiddleware next) + : this(next, null) + { + } + + public RequestTrackingMiddleware(OwinMiddleware next, TelemetryConfiguration telemetryConfiguration) + : base(next) + { + _telemetryClient = telemetryConfiguration == null + ? new TelemetryClient() + : new TelemetryClient(telemetryConfiguration); + } + + public override async Task Invoke(IOwinContext context) + { + var requestId = context.Get(OwinRequestIdKey); + + var requestMethod = context.Request.Method; + var requestPath = context.Request.Path.ToString(); + var requestUri = context.Request.Uri; + + var requestStartDate = DateTimeOffset.Now; + var stopWatch = new Stopwatch(); + stopWatch.Start(); + + var requestFailed = false; + + try + { + OwinRequestIdContext.Set(requestId); + + if (Next != null) + { + await Next.Invoke(context); + } + } + catch (Exception ex) + { + requestFailed = true; + + _telemetryClient.TrackException(ex); + + throw; + } + finally + { + stopWatch.Stop(); + + TrackRequest(requestId, requestMethod, requestPath, requestUri, + context.Response?.StatusCode ?? 0, requestFailed, requestStartDate, stopWatch.Elapsed); + + OwinRequestIdContext.Clear(); + } + } + + private void TrackRequest(string requestId, string requestMethod, string path, Uri uri, int responseCode, + bool requestFailed, DateTimeOffset requestStartDate, TimeSpan duration) + { + var name = $"{requestMethod} {path}"; + + var telemetry = new RequestTelemetry + { + Id = requestId, + Name = name, + Timestamp = requestStartDate, + Duration = duration, + ResponseCode = responseCode.ToString(), + Success = (responseCode >= 200 && responseCode <= 299) || !requestFailed, + Url = uri + }; + + telemetry.Context.Operation.Name = name; + + _telemetryClient.TrackRequest(telemetry); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Jobs.Auxiliary2AzureSearch/App.config b/src/NuGet.Jobs.Auxiliary2AzureSearch/App.config new file mode 100644 index 000000000..56efbc7b5 --- /dev/null +++ b/src/NuGet.Jobs.Auxiliary2AzureSearch/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/NuGet.Jobs.Auxiliary2AzureSearch/Job.cs b/src/NuGet.Jobs.Auxiliary2AzureSearch/Job.cs new file mode 100644 index 000000000..69f6718f0 --- /dev/null +++ b/src/NuGet.Jobs.Auxiliary2AzureSearch/Job.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using NuGet.Services.AzureSearch; +using NuGet.Services.AzureSearch.Auxiliary2AzureSearch; +using NuGet.Services.AzureSearch.AuxiliaryFiles; + +namespace NuGet.Jobs +{ + public class Job : AzureSearchJob + { + private const string ConfigurationSectionName = "Auxiliary2AzureSearch"; + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + base.ConfigureJobServices(services, configurationRoot); + + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + } + } +} diff --git a/src/NuGet.Jobs.Auxiliary2AzureSearch/NuGet.Jobs.Auxiliary2AzureSearch.csproj b/src/NuGet.Jobs.Auxiliary2AzureSearch/NuGet.Jobs.Auxiliary2AzureSearch.csproj new file mode 100644 index 000000000..3c3b83b86 --- /dev/null +++ b/src/NuGet.Jobs.Auxiliary2AzureSearch/NuGet.Jobs.Auxiliary2AzureSearch.csproj @@ -0,0 +1,81 @@ + + + + + + Debug + AnyCPU + {7E6903A4-DBE1-444E-A8E3-C1DBB58243E0} + Exe + NuGet.Jobs + NuGet.Jobs.Auxiliary2AzureSearch + v4.7.2 + 512 + true + true + PackageReference + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + 0.3.0 + runtime; build; native; contentfiles; analyzers + all + + + + + {4B4B1EFB-8F33-42E6-B79F-54E7F3293D31} + NuGet.Jobs.Common + + + {1a53fe3d-8041-4773-942f-d73aef5b82b2} + NuGet.Services.AzureSearch + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + + + \ No newline at end of file diff --git a/src/NuGet.Jobs.Auxiliary2AzureSearch/NuGet.Jobs.Auxiliary2AzureSearch.nuspec b/src/NuGet.Jobs.Auxiliary2AzureSearch/NuGet.Jobs.Auxiliary2AzureSearch.nuspec new file mode 100644 index 000000000..112c92f02 --- /dev/null +++ b/src/NuGet.Jobs.Auxiliary2AzureSearch/NuGet.Jobs.Auxiliary2AzureSearch.nuspec @@ -0,0 +1,16 @@ + + + + Auxiliary2AzureSearch + $version$ + .NET Foundation + .NET Foundation + Auxiliary2AzureSearch + Copyright .NET Foundation + + + + + + + \ No newline at end of file diff --git a/src/NuGet.Jobs.Auxiliary2AzureSearch/Program.cs b/src/NuGet.Jobs.Auxiliary2AzureSearch/Program.cs new file mode 100644 index 000000000..a87c79434 --- /dev/null +++ b/src/NuGet.Jobs.Auxiliary2AzureSearch/Program.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Jobs +{ + public class Program + { + public static int Main(string[] args) + { + var job = new Job(); + return JobRunner.Run(job, args).GetAwaiter().GetResult(); + } + } +} diff --git a/src/NuGet.Jobs.Auxiliary2AzureSearch/Properties/AssemblyInfo.cs b/src/NuGet.Jobs.Auxiliary2AzureSearch/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..8a4b8057d --- /dev/null +++ b/src/NuGet.Jobs.Auxiliary2AzureSearch/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Jobs.Auxiliary2AzureSearch")] +[assembly: ComVisible(false)] +[assembly: Guid("7e6903a4-dbe1-444e-a8e3-c1dbb58243e0")] diff --git a/src/NuGet.Jobs.Auxiliary2AzureSearch/README.md b/src/NuGet.Jobs.Auxiliary2AzureSearch/README.md new file mode 100644 index 000000000..a8c509fa7 --- /dev/null +++ b/src/NuGet.Jobs.Auxiliary2AzureSearch/README.md @@ -0,0 +1,111 @@ +## Overview + +**Subsystem: Search 🔎** + +This job updates the [search auxiliary files](../../docs/Search-auxiliary-files.md) used by the [search service](../NuGet.Services.SearchService). This also updates the downloads and owners data in the [Azure Search "search" index](../../docs/Azure-Search-indexes.md). + +## Running the job + +You can run this job using: + +```ps1 +NuGet.Jobs.Auxiliary2AzureSearch.exe -Configuration path\to\your\settings.json +``` + +This job is a singleton. Only a single instance of the job should be running per Azure Search resource. + +### Using DEV resources + +The easiest way to run the tool if you are on the nuget.org team is to use the DEV environment resources: + +1. Install the certificate used to authenticate as our client AAD app registration into your `CurrentUser` certificate store. +1. Clone our internal [`NuGetDeployment`](https://nuget.visualstudio.com/DefaultCollection/NuGetMicrosoft/_git/NuGetDeploymentp) repository. +1. Update your cloned copy of the [DEV Auxiliary2AzureSearch appsettings.json](https://nuget.visualstudio.com/DefaultCollection/NuGetMicrosoft/_git/NuGetDeployment?path=%2Fsrc%2FJobs%2FNuGet.Jobs.Cloud%2FJobs%2FAuxiliary2AzureSearch%2FDEV%2Fnorthcentralus%2Fa%2Fappsettings.json) file to authenticate using the certificate you installed: +```json +{ + ... + "KeyVault_VaultName": "PLACEHOLDER", + "KeyVault_ClientId": "PLACEHOLDER", + "KeyVault_CertificateThumbprint": "PLACEHOLDER", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "CurrentUser" + ... +} +``` + +1. Update the `-Configuration` CLI option to point to the DEV Azure Search settings: `NuGetDeployment/src/Jobs/NuGet.Jobs.Cloud/Jobs/Auxiliary2AzureSearch/DEV/northcentralus/a/appsettings.json` + +### Using personal Azure resources + +As an alternative to using nuget.org's DEV resources, you can also run this tool using your personal Azure resources. + +#### Prerequisites + +Run the [`Db2AzureSearch`](../NuGet.Jobs.Db2AzureSearch) tool. + +#### Settings + +Once you've created your Azure resources, you can create your `settings.json` file. There's a few `PLACEHOLDER` values you will need to fill in yourself: + +* The `GalleryDb:ConnectionString` setting is the connection string to your Gallery DB. +* The `SearchServiceName` setting is the name of your Azure Search resource. For example, use the name `foo-bar` for the Azure Search service with URL `https://foo-bar.search.windows.net`. +* The `SearchServiceApiKey` setting is an admin key that has write permissions to the Azure Search resource. +* The `AuxiliaryDataStorageContainer` and `StorageConnectionString` settings are the connection strings to your Azure Blob Storage account. + +```json +{ + "GalleryDb": { + "ConnectionString": "PLACEHOLDER" + }, + + "Auxiliary2AzureSearch": { + "AzureSearchBatchSize": 1000, + "MaxConcurrentBatches": 1, + "MaxConcurrentVersionListWriters": 32, + "SearchServiceName": "PLACEHOLDER", + "SearchServiceApiKey": "PLACEHOLDER", + "SearchIndexName": "search-000", + "HijackIndexName": "hijack-000", + "StorageConnectionString": "PLACEHOLDER", + "StorageContainer": "v3-azuresearch-000", + "StoragePath": "", + "AuxiliaryDataStorageConnectionString": "PLACEHOLDER", + "AuxiliaryDataStorageContainer": "ng-search-data", + "AuxiliaryDataStorageDownloadsPath": "downloads.v1.json", + "AuxiliaryDataStorageExcludedPackagesPath": "ExcludedPackages.v1.json", + "AuxiliaryDataStorageVerifiedPackagesPath": "verifiedPackages.json", + "MinPushPeriod": "00:00:10", + "MaxDownloadCountDecreases": 30000, + "EnablePopularityTransfers": true, + "Scoring": { + "PopularityTransfer": 0.99 + } + } +} +``` + +## Algorithm + +At a high-level, here's how Auxiliary2AzureSearch works: + +1. Update verified packages + 1. Get the "old" list of verified package IDs from search auxiliary storage + 2. Get the "new" list of verified package IDs from Gallery DB + 3. Replace the verified package auxiliary file if needed +1. Update downloads + 1. Get the "old" downloads data from search auxiliary storage + 1. Get the "new" downloads data from statistics auxiliary storage + 1. Determine which packages have download changes + 1. Get the "old" popularity transfers data from search auxiliary storage + 1. Get the "new" popularity transfers data from statistics auxiliary storage + 1. Determine which packages have popularity transfer changes + 1. Update Azure Search documents in the "search" index to reflect the latest downloads and popularity transfers + 1. Save the "new" downloads data to the search auxiliary storage + 1. Save the "new" popularity transfers data to search auxiliary storage +1. Update owners + 1. Get the "old" owners data from search auxiliary storage + 1. Get the "new" owners data from Gallery DB + 1. Update Azure Search documents in the "search" index to reflect the ownership changes, if any + 1. Track ownership changes in search auxiliary storage + 1. Save the "new" owners data to the search auxiliary storage \ No newline at end of file diff --git a/src/NuGet.Jobs.Auxiliary2AzureSearch/Scripts/Functions.ps1 b/src/NuGet.Jobs.Auxiliary2AzureSearch/Scripts/Functions.ps1 new file mode 100644 index 000000000..a8bff40fc --- /dev/null +++ b/src/NuGet.Jobs.Auxiliary2AzureSearch/Scripts/Functions.ps1 @@ -0,0 +1,30 @@ +Function Uninstall-NuGetService() { + Param ([string]$ServiceName) + + if (Get-Service $ServiceName -ErrorAction SilentlyContinue) + { + Write-Host Removing service $ServiceName... + Stop-Service $ServiceName -Force + sc.exe delete $ServiceName + Write-Host Removed service $ServiceName. + } else { + Write-Host Skipping removal of service $ServiceName - no such service exists. + } +} + +Function Install-NuGetService() { + Param ([string]$ServiceName, [string]$ServiceTitle, [string]$ScriptToRun) + + Write-Host Installing service $ServiceName... + + $installService = "nssm install $ServiceName $ScriptToRun" + cmd /C $installService + + Set-Service -Name $ServiceName -DisplayName "$ServiceTitle - $ServiceName" -Description "Runs $ServiceTitle." -StartupType Automatic + sc.exe failure $ServiceName reset= 30 actions= restart/5000 + + # Run service + net start $ServiceName + + Write-Host Installed service $ServiceName. +} \ No newline at end of file diff --git a/src/NuGet.Jobs.Auxiliary2AzureSearch/Scripts/PostDeploy.ps1 b/src/NuGet.Jobs.Auxiliary2AzureSearch/Scripts/PostDeploy.ps1 new file mode 100644 index 000000000..7d5183d5b --- /dev/null +++ b/src/NuGet.Jobs.Auxiliary2AzureSearch/Scripts/PostDeploy.ps1 @@ -0,0 +1,18 @@ +. .\Functions.ps1 + +$jobsToInstall = $OctopusParameters["Jobs.ServiceNames"].Split("{,}") + +Write-Host Installing services... + +$currentDirectory = [string](Get-Location) + +$jobsToInstall.Split("{;}") | %{ + $serviceName = $_ + $serviceTitle = $OctopusParameters["Jobs.$serviceName.Title"] + $scriptToRun = $OctopusParameters["Jobs.$serviceName.Script"] + $scriptToRun = "$currentDirectory\$scriptToRun" + + Install-NuGetService -ServiceName $serviceName -ServiceTitle $serviceTitle -ScriptToRun $scriptToRun +} + +Write-Host Installed services. \ No newline at end of file diff --git a/src/NuGet.Jobs.Auxiliary2AzureSearch/Scripts/PreDeploy.ps1 b/src/NuGet.Jobs.Auxiliary2AzureSearch/Scripts/PreDeploy.ps1 new file mode 100644 index 000000000..ef711a912 --- /dev/null +++ b/src/NuGet.Jobs.Auxiliary2AzureSearch/Scripts/PreDeploy.ps1 @@ -0,0 +1,11 @@ +. .\Functions.ps1 + +$jobsToInstall = $OctopusParameters["Jobs.ServiceNames"].Split("{,}") + +Write-Host Removing services... + +$jobsToInstall.Split("{;}") | %{ + Uninstall-NuGetService -ServiceName $_ +} + +Write-Host Removed services. \ No newline at end of file diff --git a/src/NuGet.Jobs.Auxiliary2AzureSearch/Scripts/nssm.exe b/src/NuGet.Jobs.Auxiliary2AzureSearch/Scripts/nssm.exe new file mode 100644 index 000000000..6ccfe3cfb Binary files /dev/null and b/src/NuGet.Jobs.Auxiliary2AzureSearch/Scripts/nssm.exe differ diff --git a/src/NuGet.Jobs.Catalog2AzureSearch/App.config b/src/NuGet.Jobs.Catalog2AzureSearch/App.config new file mode 100644 index 000000000..56efbc7b5 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2AzureSearch/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2AzureSearch/Job.cs b/src/NuGet.Jobs.Catalog2AzureSearch/Job.cs new file mode 100644 index 000000000..7839b6cf2 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2AzureSearch/Job.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using NuGet.Services.AzureSearch; +using NuGet.Services.AzureSearch.Catalog2AzureSearch; +using NuGet.Services.V3; + +namespace NuGet.Jobs +{ + public class Job : AzureSearchJob + { + private const string ConfigurationSectionName = "Catalog2AzureSearch"; + private const string DevelopmentConfigurationSectionName = "Catalog2AzureSearch:Development"; + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + base.ConfigureJobServices(services, configurationRoot); + + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + services.Configure( + configurationRoot.GetSection(DevelopmentConfigurationSectionName)); + } + } +} diff --git a/src/NuGet.Jobs.Catalog2AzureSearch/NuGet.Jobs.Catalog2AzureSearch.csproj b/src/NuGet.Jobs.Catalog2AzureSearch/NuGet.Jobs.Catalog2AzureSearch.csproj new file mode 100644 index 000000000..646903a4d --- /dev/null +++ b/src/NuGet.Jobs.Catalog2AzureSearch/NuGet.Jobs.Catalog2AzureSearch.csproj @@ -0,0 +1,93 @@ + + + + + + Debug + AnyCPU + {F591130F-181A-4C53-8025-4390F46BD51D} + Exe + NuGet.Jobs + NuGet.Jobs.Catalog2AzureSearch + v4.7.2 + 512 + true + true + PackageReference + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + 0.3.0 + runtime; build; native; contentfiles; analyzers + all + + + + + {E97F23B8-ECB0-4AFA-B00C-015C39395FEF} + NuGet.Services.Metadata.Catalog + + + {4B4B1EFB-8F33-42E6-B79F-54E7F3293D31} + NuGet.Jobs.Common + + + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} + NuGet.Protocol.Catalog + + + {1a53fe3d-8041-4773-942f-d73aef5b82b2} + NuGet.Services.AzureSearch + + + {C3F9A738-9759-4B2B-A50D-6507B28A659B} + NuGet.Services.V3 + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + + + \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2AzureSearch/NuGet.Jobs.Catalog2AzureSearch.nuspec b/src/NuGet.Jobs.Catalog2AzureSearch/NuGet.Jobs.Catalog2AzureSearch.nuspec new file mode 100644 index 000000000..5982804ee --- /dev/null +++ b/src/NuGet.Jobs.Catalog2AzureSearch/NuGet.Jobs.Catalog2AzureSearch.nuspec @@ -0,0 +1,16 @@ + + + + Catalog2AzureSearch + $version$ + .NET Foundation + .NET Foundation + Catalog2AzureSearch + Copyright .NET Foundation + + + + + + + \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2AzureSearch/Program.cs b/src/NuGet.Jobs.Catalog2AzureSearch/Program.cs new file mode 100644 index 000000000..a87c79434 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2AzureSearch/Program.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Jobs +{ + public class Program + { + public static int Main(string[] args) + { + var job = new Job(); + return JobRunner.Run(job, args).GetAwaiter().GetResult(); + } + } +} diff --git a/src/NuGet.Jobs.Catalog2AzureSearch/Properties/AssemblyInfo.cs b/src/NuGet.Jobs.Catalog2AzureSearch/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..b6c90b7b1 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2AzureSearch/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Jobs.Catalog2AzureSearch")] +[assembly: ComVisible(false)] +[assembly: Guid("f591130f-181a-4c53-8025-4390f46bd51d")] diff --git a/src/NuGet.Jobs.Catalog2AzureSearch/README.md b/src/NuGet.Jobs.Catalog2AzureSearch/README.md new file mode 100644 index 000000000..4241d17be --- /dev/null +++ b/src/NuGet.Jobs.Catalog2AzureSearch/README.md @@ -0,0 +1,100 @@ +## Overview + +**Subsystem: Search 🔎** + +This job updates the [Azure Search indexes](../../docs/Azure-Search-indexes.md) used by the [search service](../NuGet.Services.SearchService). + +`Catalog2AzureSearch` uses the [catalog resource](https://docs.microsoft.com/en-us/nuget/api/catalog-resource) to track package events, like uploads and deletes. It also uses the [package metadata resource](https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource) to fetch packages' metadata. Finally, it tracks the latest versions of packages using the [version list resource](../../docs/Search-version-list-resource.md). + +## Running the job + +You can run this job using: + +```ps1 +NuGet.Jobs.Catalog2AzureSearch.exe -Configuration path\to\your\settings.json +``` + +This job is a singleton. Only a single instance of the job should be running per Azure Search resource. + +### Using DEV resources + +The easiest way to run the tool if you are on the nuget.org team is to use the DEV environment resources: + +1. Install the certificate used to authenticate as our client AAD app registration into your `CurrentUser` certificate store. +1. Clone our internal [`NuGetDeployment`](https://nuget.visualstudio.com/DefaultCollection/NuGetMicrosoft/_git/NuGetDeploymentp) repository. +1. Update your cloned copy of the [DEV Catalog2AzureSearch appsettings.json](https://nuget.visualstudio.com/DefaultCollection/NuGetMicrosoft/_git/NuGetDeployment?path=%2Fsrc%2FJobs%2FNuGet.Jobs.Cloud%2FJobs%2FCatalog2AzureSearch%2FDEV%2Fnorthcentralus%2Fa%2Fappsettings.json) file to authenticate using the certificate you installed: +```json +{ + ... + "KeyVault_VaultName": "PLACEHOLDER", + "KeyVault_ClientId": "PLACEHOLDER", + "KeyVault_CertificateThumbprint": "PLACEHOLDER", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "CurrentUser" + ... +} +``` + +1. Update the `-Configuration` CLI option to point to the DEV Azure Search settings: `NuGetDeployment/src/Jobs/NuGet.Jobs.Cloud/Jobs/Catalog2AzureSearch/DEV/northcentralus/a/appsettings.json` + +### Using personal Azure resources + +As an alternative to using nuget.org's DEV resources, you can also run this tool using your personal Azure resources. + +#### Prerequisites + +Run the [`Db2AzureSearch`](../NuGet.Jobs.Db2AzureSearch) tool. + +#### Settings + +Once you've created your Azure resources, you can create your `settings.json` file. There's a few `PLACEHOLDER` values you will need to fill in yourself: + +* The `GalleryDb:ConnectionString` setting is the connection string to your Gallery DB. +* The `SearchServiceName` setting is the name of your Azure Search resource. For example, use the name `foo-bar` for the Azure Search service with URL `https://foo-bar.search.windows.net`. +* The `SearchServiceApiKey` setting is an admin key that has write permissions to the Azure Search resource. +* The `StorageConnectionString` setting is the connection string to your Azure Blob Storage account. + +```json +{ + "GalleryDb": { + "ConnectionString": "PLACEHOLDER" + }, + + "Catalog2AzureSearch": { + "AzureSearchBatchSize": 1000, + "MaxConcurrentBatches": 4, + "MaxConcurrentVersionListWriters": 8, + "SearchServiceName": "PLACEHOLDER", + "SearchServiceApiKey": "PLACEHOLDER", + "SearchIndexName": "search-000", + "HijackIndexName": "hijack-000", + "StorageConnectionString": "PLACEHOLDER", + "StorageContainer": "v3-azuresearch-000", + "StoragePath": "", + "GalleryBaseUrl": "https://www.nuget.org/", + "FlatContainerBaseUrl": "https://api.nuget.org/", + "FlatContainerContainerName": "v3-flatcontainer", + "AllIconsInFlatContainer": false, + "Source": "https://api.nuget.org/v3/catalog0/index.json", + "HttpClientTimeout": "00:10:00", + "DependencyCursorUrls": [ + "https://nugetgallery.blob.core.windows.net/v3-registration5-semver1/cursor.json" + ], + "RegistrationsBaseUrl": "https://api.nuget.org/v3/registration5-gz-semver2/" + } +} +``` + +## Algorithm + +At a high-level, here's how Catalog2AzureSearch works: + +1. Load its catalog cursor from Azure Blob Storage +1. Fetch catalog leaves that are newer than the catalog cursor value +1. For each package ID in the catalog leaves: + 1. Fetch the [version list resource](../../Search-version-list-resource.md) for the package ID + 1. Apply the package's catalog leaves to the version list resource to understand which search documents need to be updated. In some cases, use the [Package Metadata resource](https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource) to fetch additional package metadata and catalog leaves + 1. Generate Azure Search actions to update the indexes +1. Push all generated Azure Search index actions +1. Save the catalog cursor to Azure Blob Storage diff --git a/src/NuGet.Jobs.Catalog2AzureSearch/Scripts/Functions.ps1 b/src/NuGet.Jobs.Catalog2AzureSearch/Scripts/Functions.ps1 new file mode 100644 index 000000000..a8bff40fc --- /dev/null +++ b/src/NuGet.Jobs.Catalog2AzureSearch/Scripts/Functions.ps1 @@ -0,0 +1,30 @@ +Function Uninstall-NuGetService() { + Param ([string]$ServiceName) + + if (Get-Service $ServiceName -ErrorAction SilentlyContinue) + { + Write-Host Removing service $ServiceName... + Stop-Service $ServiceName -Force + sc.exe delete $ServiceName + Write-Host Removed service $ServiceName. + } else { + Write-Host Skipping removal of service $ServiceName - no such service exists. + } +} + +Function Install-NuGetService() { + Param ([string]$ServiceName, [string]$ServiceTitle, [string]$ScriptToRun) + + Write-Host Installing service $ServiceName... + + $installService = "nssm install $ServiceName $ScriptToRun" + cmd /C $installService + + Set-Service -Name $ServiceName -DisplayName "$ServiceTitle - $ServiceName" -Description "Runs $ServiceTitle." -StartupType Automatic + sc.exe failure $ServiceName reset= 30 actions= restart/5000 + + # Run service + net start $ServiceName + + Write-Host Installed service $ServiceName. +} \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2AzureSearch/Scripts/PostDeploy.ps1 b/src/NuGet.Jobs.Catalog2AzureSearch/Scripts/PostDeploy.ps1 new file mode 100644 index 000000000..7d5183d5b --- /dev/null +++ b/src/NuGet.Jobs.Catalog2AzureSearch/Scripts/PostDeploy.ps1 @@ -0,0 +1,18 @@ +. .\Functions.ps1 + +$jobsToInstall = $OctopusParameters["Jobs.ServiceNames"].Split("{,}") + +Write-Host Installing services... + +$currentDirectory = [string](Get-Location) + +$jobsToInstall.Split("{;}") | %{ + $serviceName = $_ + $serviceTitle = $OctopusParameters["Jobs.$serviceName.Title"] + $scriptToRun = $OctopusParameters["Jobs.$serviceName.Script"] + $scriptToRun = "$currentDirectory\$scriptToRun" + + Install-NuGetService -ServiceName $serviceName -ServiceTitle $serviceTitle -ScriptToRun $scriptToRun +} + +Write-Host Installed services. \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2AzureSearch/Scripts/PreDeploy.ps1 b/src/NuGet.Jobs.Catalog2AzureSearch/Scripts/PreDeploy.ps1 new file mode 100644 index 000000000..ef711a912 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2AzureSearch/Scripts/PreDeploy.ps1 @@ -0,0 +1,11 @@ +. .\Functions.ps1 + +$jobsToInstall = $OctopusParameters["Jobs.ServiceNames"].Split("{,}") + +Write-Host Removing services... + +$jobsToInstall.Split("{;}") | %{ + Uninstall-NuGetService -ServiceName $_ +} + +Write-Host Removed services. \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2AzureSearch/Scripts/nssm.exe b/src/NuGet.Jobs.Catalog2AzureSearch/Scripts/nssm.exe new file mode 100644 index 000000000..6ccfe3cfb Binary files /dev/null and b/src/NuGet.Jobs.Catalog2AzureSearch/Scripts/nssm.exe differ diff --git a/src/NuGet.Jobs.Catalog2Registration/App.config b/src/NuGet.Jobs.Catalog2Registration/App.config new file mode 100644 index 000000000..ecdcf8a54 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/App.config @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/NuGet.Jobs.Catalog2Registration/Catalog2RegistrationCommand.cs b/src/NuGet.Jobs.Catalog2Registration/Catalog2RegistrationCommand.cs new file mode 100644 index 000000000..7ec4b5ba8 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Catalog2RegistrationCommand.cs @@ -0,0 +1,127 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Services.V3; +using NuGetGallery; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class Catalog2RegistrationCommand + { + public const string CursorRelativeUri = "cursor.json"; + + private readonly ICollector _collector; + private readonly ICloudBlobClient _cloudBlobClient; + private readonly IStorageFactory _storageFactory; + private readonly Func _handlerFunc; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public Catalog2RegistrationCommand( + ICollector collector, + ICloudBlobClient cloudBlobClient, + IStorageFactory storageFactory, + Func handlerFunc, + IOptionsSnapshot options, + ILogger logger) + { + _collector = collector ?? throw new ArgumentNullException(nameof(collector)); + _cloudBlobClient = cloudBlobClient ?? throw new ArgumentNullException(nameof(cloudBlobClient)); + _storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory)); + _handlerFunc = handlerFunc ?? throw new ArgumentNullException(nameof(handlerFunc)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ExecuteAsync() + { + await ExecuteAsync(CancellationToken.None); + } + + private async Task ExecuteAsync(CancellationToken token) + { + // Initialize the cursors. + ReadCursor backCursor; + if (_options.Value.DependencyCursorUrls != null + && _options.Value.DependencyCursorUrls.Any()) + { + _logger.LogInformation("Depending on cursors: {DependencyCursorUrls}", _options.Value.DependencyCursorUrls); + backCursor = new AggregateCursor(_options + .Value + .DependencyCursorUrls.Select(r => new HttpReadCursor(new Uri(r), _handlerFunc))); + } + else + { + _logger.LogInformation("Depending on no cursors, meaning the job will process up to the latest catalog information."); + backCursor = MemoryCursor.CreateMax(); + } + + var frontCursorStorage = _storageFactory.Create(); + var frontCursorUri = frontCursorStorage.ResolveUri(CursorRelativeUri); + var frontCursor = new DurableCursor(frontCursorUri, frontCursorStorage, DateTime.MinValue); + + _logger.LogInformation("Using cursor: {CursurUrl}", frontCursorUri.AbsoluteUri); + LogContainerUrl(HiveType.Legacy, c => c.LegacyStorageContainer); + LogContainerUrl(HiveType.Gzipped, c => c.GzippedStorageContainer); + LogContainerUrl(HiveType.SemVer2, c => c.SemVer2StorageContainer); + + // Optionally create the containers. + if (_options.Value.CreateContainers) + { + await CreateContainerIfNotExistsAsync(c => c.LegacyStorageContainer); + await CreateContainerIfNotExistsAsync(c => c.GzippedStorageContainer); + await CreateContainerIfNotExistsAsync(c => c.SemVer2StorageContainer); + } + + await frontCursor.LoadAsync(token); + await backCursor.LoadAsync(token); + _logger.LogInformation( + "The cursors have been loaded. Front: {FrontCursor}. Back: {BackCursor}.", + frontCursor.Value, + backCursor.Value); + + // Run the collector. + await _collector.RunAsync( + frontCursor, + backCursor, + token); + } + + private async Task CreateContainerIfNotExistsAsync(Func getContainer) + { + var containerName = getContainer(_options.Value); + var container = _cloudBlobClient.GetContainerReference(containerName); + + _logger.LogInformation("Creating container {Container} if it does not already exist.", containerName); + await container.CreateIfNotExistAsync(); + _logger.LogInformation("Ensuring container {Container} has blobs publicly available.", containerName); + await container.SetPermissionsAsync(new BlobContainerPermissions + { + PublicAccess = BlobContainerPublicAccessType.Blob, + }); + } + + private void LogContainerUrl(HiveType hive, Func getContainer) + { + _logger.LogInformation( + "Using {Hive} storage: {ContainerUrl}", + hive, + CloudStorageAccount.Parse(_options.Value.StorageConnectionString) + .CreateCloudBlobClient() + .GetContainerReference(getContainer(_options.Value)) + .Uri + .AbsoluteUri); + } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Catalog2RegistrationConfiguration.cs b/src/NuGet.Jobs.Catalog2Registration/Catalog2RegistrationConfiguration.cs new file mode 100644 index 000000000..f02a35d34 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Catalog2RegistrationConfiguration.cs @@ -0,0 +1,140 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Services.V3; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class Catalog2RegistrationConfiguration : ICommitCollectorConfiguration + { + private static readonly int DefaultMaxConcurrentHivesPerId = Enum.GetValues(typeof(HiveType)).Length; + + /// + /// The connection string used to connect to an Azure Blob Storage account. The connection string specifies + /// the account name, the endpoint suffix (e.g. Azure vs. Azure China), and authentication credential (e.g. storage + /// key). + /// + public string StorageConnectionString { get; set; } + + /// + /// The blob storage container for the legacy hive (not gzipped, no SemVer 2.0.0 packages). This container is in + /// the Azure Blob Storage account specified in . + /// + public string LegacyStorageContainer { get; set; } + + /// + /// The user-facing base URL for the legacy registration hive. + /// + public string LegacyBaseUrl { get; set; } + + /// + /// The blob storage container for the gzipped hive (no SemVer 2.0.0 packages). This container is in the Azure + /// Blob Storage account specified in . + /// + public string GzippedStorageContainer { get; set; } + + /// + /// The user-facing base URL for the gzipped registration hive. + /// + public string GzippedBaseUrl { get; set; } + + /// + /// The blob storage container for the SemVer 2.0.0 hive (gzipped and SemVer 2.0.0 packages). This container is + /// in the Azure Blob Storage account specified in . + /// + public string SemVer2StorageContainer { get; set; } + + /// + /// The user-facing base URL for the SemVer 2.0.0 registration hive. + /// + public string SemVer2BaseUrl { get; set; } + + /// + /// Zero or more URL to the dependency cursor URLs. The registration collector will go no further in the + /// catalog than these cursors. + /// + public List DependencyCursorUrls { get; set; } + + /// + /// The catalog index URL to poll for package details and package deletes. + /// + public string Source { get; set; } + + /// + /// The flat container base URL (i.e. the package base address) as it appears in the service index. This will + /// be used to create user-facing URLs so should match the service index. This will be used for generating + /// .nupkg and package icon URLs. + /// + public string FlatContainerBaseUrl { get; set; } + + /// + /// The gallery base URL. This will be used for generating package license URLs. + /// + public string GalleryBaseUrl { get; set; } + + /// + /// The maximum number of catalog leafs to download in parallel. When a batch of new catalog leafs is found + /// in the catalog, the package details leaves for all package IDs are downloaded in parallel. Package delete + /// leaves are not downloaded and therefore are not relevant to this setting. + /// + public int MaxConcurrentCatalogLeafDownloads { get; set; } = 64; + + /// + /// The timeout used for the collector . + /// + public TimeSpan HttpClientTimeout { get; set; } = TimeSpan.FromMinutes(10); + + /// + /// Whether or not the registration containers should be created at runtime. In general it is best to allow + /// ops tools or manual process to create containers so that the public access level can be properly set. + /// + public bool CreateContainers { get; set; } + + /// + /// The maximum number of package IDs to process in parallel. This parallelism + /// can be constrained by as well. + /// + public int MaxConcurrentIds { get; set; } = 64; + + /// + /// The maximum number of hives to process in parallel for a single ID. The other parallelism controls are + /// probably more interesting so this setting should probably be left at the default (full parallelism) unless + /// you are trying to force sequential processing in which case it would be set to 1. + /// + public int MaxConcurrentHivesPerId { get; set; } = DefaultMaxConcurrentHivesPerId; + + /// + /// The maximum number of asynchronous operations to perform while processing a single hive. This parallelism + /// can be constrained by as well. + /// + public int MaxConcurrentOperationsPerHive { get; set; } = 64; + + /// + /// The maximum number of blob storage operations (read, write, delete) that can be performed in parallel. + /// + public int MaxConcurrentStorageOperations { get; set; } = 64; + + /// + /// The maximum number of registration leaf items to allow in a registration page. If the number of items + /// exceeds this number, a new page item will be created. + /// + public int MaxLeavesPerPage { get; set; } = 64; + + /// + /// The maximum number of leaf items to allow before pages stop being inlined. In other words, if there are + /// less than or equal to this number of package versions for single ID, pages will be inlined in the + /// registration index. If there are more than this number of package versions for a single ID, pages will be + /// written to their own blobs (i.e. they will no longer be inlined in the index). + /// + public int MaxInlinedLeafItems { get; set; } = 127; + + /// + /// Whenever a blob storage write is performed, the code will also ensure that there is at least a single + /// snapshot. The snapshot's content is not important but is present to mitigate accidental deletion via + /// tooling such as Azure Storage Explorer. + /// + public bool EnsureSingleSnapshot { get; set; } = true; + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/DependencyInjectionExtensions.cs b/src/NuGet.Jobs.Catalog2Registration/DependencyInjectionExtensions.cs new file mode 100644 index 000000000..de3ffa949 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/DependencyInjectionExtensions.cs @@ -0,0 +1,103 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net.Http; +using Autofac; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using NuGet.Protocol; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Services.V3; +using NuGetGallery; + +namespace NuGet.Jobs.Catalog2Registration +{ + public static class DependencyInjectionExtensions + { + public const string CursorBindingKey = "Cursor"; + + public static ContainerBuilder AddCatalog2Registration(this ContainerBuilder containerBuilder) + { + RegisterCursorStorage(containerBuilder); + + containerBuilder + .Register(c => + { + var options = c.Resolve>(); + return new CloudBlobClientWrapper( + options.Value.StorageConnectionString, + DefaultBlobRequestOptions.Create()); + }); + + containerBuilder.Register(c => new Catalog2RegistrationCommand( + c.Resolve(), + c.Resolve(), + c.ResolveKeyed(CursorBindingKey), + c.Resolve>(), + c.Resolve>(), + c.Resolve>())); + + return containerBuilder; + } + + private static void RegisterCursorStorage(ContainerBuilder containerBuilder) + { + // Register NuGet.Services.Metadata storage abstractions with a binding key so that they are not as easy to + // consume. This is an intentional decision since product code should use the NuGetGallery.Core storage + // abstractions (e.g. ICloudBlobClient). + containerBuilder + .Register(c => + { + var options = c.Resolve>(); + return CloudStorageAccount.Parse(options.Value.StorageConnectionString); + }) + .Keyed(CursorBindingKey); + + containerBuilder + .Register(c => + { + var options = c.Resolve>(); + return new AzureStorageFactory( + c.ResolveKeyed(CursorBindingKey), + options.Value.LegacyStorageContainer, + maxExecutionTime: AzureStorage.DefaultMaxExecutionTime, + serverTimeout: AzureStorage.DefaultServerTimeout, + path: string.Empty, + baseAddress: null, + useServerSideCopy: true, + compressContent: false, + verbose: true, + initializeContainer: false, + throttle: NullThrottle.Instance); + }) + .Keyed(CursorBindingKey); + } + + public static IServiceCollection AddCatalog2Registration( + this IServiceCollection services, + IDictionary telemetryGlobalDimensions) + { + services.AddV3(telemetryGlobalDimensions); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.AddSingleton(s => + { + var config = s.GetRequiredService>(); + return SemaphoreSlimThrottle.CreateSemaphoreThrottle(config.Value.MaxConcurrentStorageOperations); + }); + + return services; + } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Hives/Bookkeeping/IndexInfo.cs b/src/NuGet.Jobs.Catalog2Registration/Hives/Bookkeeping/IndexInfo.cs new file mode 100644 index 000000000..451fc5fbf --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Hives/Bookkeeping/IndexInfo.cs @@ -0,0 +1,80 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using NuGet.Protocol.Registration; + +namespace NuGet.Jobs.Catalog2Registration +{ + /// + /// A class that handles the bookkeeping of modifying a registration index. It holds a reference to a + /// that can be later serialized into a blob. It also holds references to + /// bookkeeping objects for its contained pages. + /// + [DebuggerDisplay("{DebuggerDisplay,nq}")] + public class IndexInfo + { + private string DebuggerDisplay => $"Index ({Items.Count} page items)"; + + private readonly List _items; + + private IndexInfo(RegistrationIndex index, List items) + { + Index = index ?? throw new ArgumentNullException(nameof(index)); + _items = items ?? throw new ArgumentNullException(nameof(items)); + } + + public static IndexInfo Existing(IHiveStorage storage, HiveType hive, RegistrationIndex index) + { + // Ensure the index is sorted in ascending order by lower version bound. + var sorted = index.Items + .Select(pageItem => new + { + PageItem = pageItem, + PageInfo = PageInfo.Existing(pageItem, url => GetPageAsync(storage, hive, url)), + }) + .OrderBy(x => x.PageInfo.Lower) + .ToList(); + + var items = sorted.Select(x => x.PageInfo).ToList(); + index.Items.Clear(); + index.Items.AddRange(sorted.Select(x => x.PageItem)); + + return new IndexInfo(index, items); + } + + private static async Task GetPageAsync(IHiveStorage storage, HiveType hive, string url) + { + return await storage.ReadPageAsync(hive, url); + } + + public static IndexInfo New() + { + var index = new RegistrationIndex + { + Items = new List(), + }; + + return new IndexInfo(index, new List()); + } + + public RegistrationIndex Index { get; } + public IReadOnlyList Items => _items; + + public void RemoveAt(int index) + { + _items.RemoveAt(index); + Index.Items.RemoveAt(index); + } + + public void Insert(int index, PageInfo pageInfo) + { + _items.Insert(index, pageInfo); + Index.Items.Insert(index, pageInfo.PageItem); + } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Hives/Bookkeeping/LeafInfo.cs b/src/NuGet.Jobs.Catalog2Registration/Hives/Bookkeeping/LeafInfo.cs new file mode 100644 index 000000000..d7e37cdd4 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Hives/Bookkeeping/LeafInfo.cs @@ -0,0 +1,47 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using NuGet.Protocol.Registration; +using NuGet.Versioning; + +namespace NuGet.Jobs.Catalog2Registration +{ + /// + /// A class that handled the bookkeeping of a leaf item. The leaf item has minimal bookkeeping required except + /// maintaining a parsed version object for easy comparison. + /// + [DebuggerDisplay("{DebuggerDisplay,nq}")] + public class LeafInfo + { + private string DebuggerDisplay => $"Leaf {Version.ToNormalizedString()}"; + + private LeafInfo(NuGetVersion version, RegistrationLeafItem leafItem) + { + Version = version ?? throw new ArgumentNullException(nameof(version)); + LeafItem = leafItem ?? throw new ArgumentNullException(nameof(leafItem)); + } + + public static LeafInfo New(NuGetVersion version) + { + return new LeafInfo(version, new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Version = version.ToFullString(), + } + }); + } + + public static LeafInfo Existing(RegistrationLeafItem leafItem) + { + return new LeafInfo( + NuGetVersion.Parse(leafItem.CatalogEntry.Version), + leafItem); + } + + public NuGetVersion Version { get; } + public RegistrationLeafItem LeafItem { get; } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Hives/Bookkeeping/PageInfo.cs b/src/NuGet.Jobs.Catalog2Registration/Hives/Bookkeeping/PageInfo.cs new file mode 100644 index 000000000..2ed7e0418 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Hives/Bookkeeping/PageInfo.cs @@ -0,0 +1,234 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using NuGet.Protocol.Registration; +using NuGet.Services; +using NuGet.Versioning; + +namespace NuGet.Jobs.Catalog2Registration +{ + /// + /// A class that handles the bookkeeping of a registration page. This contains the page item for easy access but + /// also has the logic to fetch the leaf items if they are not inlined. + /// + /// The count and bounds properties are updated by the and + /// methods as items are moved around but note that the only property on the + /// (or external page) updated by this bookkeeping class is + /// . The other properties can be updated by the caller prior to serialization + /// as necessary. + /// + /// The main benefit of a dedicated bookkeeping class is that it can hold parsed version instances and can keep + /// them up to date given relevant state-changing actions. For example, removing a leaf item changes the count and + /// can change the bounds. + /// + /// In general pages are ephemeral things that don't hold any unique state than can't be inferred by the contained + /// leaf items. + /// + [DebuggerDisplay("{DebuggerDisplay,nq}")] + public class PageInfo + { + private string DebuggerDisplay => + $"Page " + + $"[{Lower?.ToNormalizedString() ?? "null"}, {Upper?.ToNormalizedString() ?? "null"}] " + + $"({Count} leaf items)"; + + private readonly Lazy _lazyInitializeTask; + private RegistrationPage _page; + private List _leafInfos; + + private PageInfo( + RegistrationPage pageItem, + Func initializeTaskFactory) + { + PageItem = pageItem ?? throw new ArgumentNullException(nameof(pageItem)); + + if (initializeTaskFactory == null) + { + throw new ArgumentNullException(nameof(initializeTaskFactory)); + } + + _lazyInitializeTask = new Lazy(() => initializeTaskFactory(this)); + } + + public static PageInfo New() + { + var pageItem = new RegistrationPage + { + Items = new List(), + }; + + var pageInfo = new PageInfo(pageItem, _ => Task.CompletedTask); + Initialize(pageInfo, pageItem); + return pageInfo; + } + + public static PageInfo Existing( + RegistrationPage pageItem, + Func> getPageByUrlAsync) + { + if (pageItem.Items == null) + { + if (getPageByUrlAsync == null) + { + throw new ArgumentNullException(nameof(getPageByUrlAsync)); + } + + return new PageInfo( + pageItem, + async pageInfo => + { + var page = await getPageByUrlAsync(pageItem.Url); + Initialize(pageInfo, page); + }) + { + Count = pageItem.Count, + Lower = NuGetVersion.Parse(pageItem.Lower), + Upper = NuGetVersion.Parse(pageItem.Upper), + }; + } + else + { + var pageInfo = new PageInfo(pageItem, _ => Task.CompletedTask); + Initialize(pageInfo, pageItem); + return pageInfo; + } + } + + public bool IsInlined => PageItem.Items != null; + public int Count { get; private set; } + public NuGetVersion Lower { get; private set; } + public NuGetVersion Upper { get; private set; } + public RegistrationPage PageItem { get; set; } + public bool IsPageFetched => _lazyInitializeTask.IsValueCreated; + + private static void Initialize(PageInfo pageInfo, RegistrationPage page) + { + // Ensure the page is sorted in ascending order by version. + var leafInfos = page + .Items + .Select(x => LeafInfo.Existing(x)) + .OrderBy(x => x.Version) + .ToList(); + page.Items.Clear(); + page.Items.AddRange(leafInfos.Select(x => x.LeafItem)); + + // Update the bookkeeping with the latest information. The leaf items themselves are the "true" for the + // count, lower bound, and upper bound properties not whatever might be set on the page item. + pageInfo._page = page; + pageInfo._leafInfos = leafInfos; + pageInfo.Count = page.Items.Count; + pageInfo.Lower = leafInfos.FirstOrDefault()?.Version; + pageInfo.Upper = leafInfos.LastOrDefault()?.Version; + } + + /// + /// Clone this page info into another instance but with the leaf items inlined. An inlined page is one that has + /// its leaf items in the page item, not in an external page. + /// + /// The new page info instance with leaf items inlined. + public async Task CloneToInlinedAsync() + { + var page = await GetPageAsync(); + return Existing(page, getPageByUrlAsync: null); + } + + /// + /// Clone this page info into another instance but with the leaf items not inlined. A non-inlined page is one + /// that has a null item list in the page item. The leaf items are stored in an external page. + /// + /// The new page info instance with leaf items in a different page instance. + public async Task CloneToNonInlinedAsync() + { + var page = await GetPageAsync(); + var pageInfo = new PageInfo(new RegistrationPage(), _ => Task.CompletedTask); + Initialize(pageInfo, page); + return pageInfo; + } + + public async Task RemoveAtAsync(int index) + { + var page = await GetPageAsync(); + var leafInfos = await GetMutableLeafInfosAsync(); + + // Remove from the real page, for future serialization. + page.Items.RemoveAt(index); + + // Remove from the leaf info list, for bookkeeping. + var leafInfo = leafInfos[index]; + leafInfos.RemoveAt(index); + + Count--; + UpdateBounds(page, leafInfos); + + return leafInfo; + } + + public async Task InsertAsync(int index, LeafInfo leafInfo) + { + var page = await GetPageAsync(); + var leafInfos = await GetMutableLeafInfosAsync(); + + if (index > 0) + { + Guard.Assert( + leafInfos[index - 1].Version < leafInfo.Version, + "The version added to a page must have a higher version than the item before it."); + } + + if (index < leafInfos.Count) + { + Guard.Assert( + leafInfos[index].Version > leafInfo.Version, + "The version added to a page must have a lower version than the item after it."); + } + + // Add to the real page, for future serialization. + page.Items.Insert(index, leafInfo.LeafItem); + + // Add to the leaf info list, for bookeeping. + leafInfos.Insert(index, leafInfo); + + Count++; + UpdateBounds(page, leafInfos); + } + + private void UpdateBounds(RegistrationPage page, List leafInfos) + { + Guard.Assert(Count == page.Items.Count, "The count property on the page info must match the number of leaf items."); + Guard.Assert(Count == leafInfos.Count, "The count property on the page info match the number of leaf infos."); + + if (Count > 0) + { + Lower = leafInfos.First().Version; + Upper = leafInfos.Last().Version; + } + else + { + Lower = null; + Upper = null; + } + } + + public async Task GetPageAsync() + { + await _lazyInitializeTask.Value; + return _page; + } + + public async Task> GetLeafInfosAsync() + { + return await GetMutableLeafInfosAsync(); + } + + private async Task> GetMutableLeafInfosAsync() + { + await _lazyInitializeTask.Value; + return _leafInfos; + } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Hives/CatalogCommit.cs b/src/NuGet.Jobs.Catalog2Registration/Hives/CatalogCommit.cs new file mode 100644 index 000000000..548783803 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Hives/CatalogCommit.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class CatalogCommit + { + public CatalogCommit(string id, DateTimeOffset timestamp) + { + Id = id; + Timestamp = timestamp; + } + + public string Id { get; } + public DateTimeOffset Timestamp { get; } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Hives/HiveMergeResult.cs b/src/NuGet.Jobs.Catalog2Registration/Hives/HiveMergeResult.cs new file mode 100644 index 000000000..91a2950fc --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Hives/HiveMergeResult.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class HiveMergeResult + { + public HiveMergeResult(HashSet modifiedPages, HashSet modifiedLeaves, HashSet deletedLeaves) + { + ModifiedPages = modifiedPages; + ModifiedLeaves = modifiedLeaves; + DeletedLeaves = deletedLeaves; + } + + public HashSet ModifiedPages { get; } + public HashSet ModifiedLeaves { get; } + public HashSet DeletedLeaves { get; } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Hives/HiveMerger.cs b/src/NuGet.Jobs.Catalog2Registration/Hives/HiveMerger.cs new file mode 100644 index 000000000..64e83cd8c --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Hives/HiveMerger.cs @@ -0,0 +1,312 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Services; +using NuGet.Services.Metadata.Catalog; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class HiveMerger : IHiveMerger + { + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public HiveMerger( + IOptionsSnapshot options, + ILogger logger) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + private int MaxLeavesPerPage => _options.Value.MaxLeavesPerPage; + + public async Task MergeAsync(IndexInfo indexInfo, IReadOnlyList sortedCatalog) + { + for (var i = 1; i < sortedCatalog.Count; i++) + { + Guard.Assert( + sortedCatalog[i - 1].PackageIdentity.Version < sortedCatalog[i].PackageIdentity.Version, + "The catalog commit items must be in ascending order by version."); + } + + var context = new Context(indexInfo, sortedCatalog); + + await MergeAsync(context); + + return new HiveMergeResult( + context.ModifiedPages, + context.ModifiedLeaves, + context.DeletedLeaves); + } + + private async Task MergeAsync(Context context) + { + // The general approach here is to use the merge algorithm to combine the incoming list of catalog commit + // items with the existing list of registration leaf items to end up with a new sorted list of registration + // leaf items. There are additional complexities in addition a textbook "merge" algorithm: + // + // 1. Items from the catalog of type PackageDelete can result in a removal from the result list. + // + // 2. Items from the catalog can be ignored if it is a PackageDelete on a version that does not exist in + // the list of registration leaf items. + // + // 3. When both input sorted lists (catalog and registration) have the same version, the item from the + // catalog replaces the item from the registration instead of having both versions next to each other. + // + // 4. There is additional bookkeeping requires to manage pages. We have a maximum number of leaves per + // registration page so we need to handle cases when a full page needs to be filled up again after + // a package delete or have items pushed to a later page if it is too full (from a package insert). + // + // These are just complexities though. The general approach of "merge" still works fine. The catalog leaves + // can be sorted by version in memory and the registration leaves are already sorted. We will iterate + // through the two lists in parallel by using queue data structures. + // + // The benefit of the merge algorithm is that we don't need to touch registration pages that don't bound + // any of the incoming catalog leaves. + // + // See the wonderful Wikipedia page: https://en.wikipedia.org/wiki/Merge_algorithm + + var catalogIndex = 0; + var pageIndex = 0; + + var sortedCatalog = context.SortedCatalog; + var pageInfos = context.IndexInfo.Items; + + // Keep track of the position in the current page. + var itemIndex = 0; + + // This is the main "merge" step where the two input lists are interleaved. + while (catalogIndex < sortedCatalog.Count && pageIndex < pageInfos.Count) + { + var catalog = sortedCatalog[catalogIndex]; + var pageInfo = pageInfos[pageIndex]; + var nextLower = pageIndex < pageInfos.Count - 1 ? pageInfos[pageIndex + 1].Lower : null; + var hasGap = pageInfo.Count < MaxLeavesPerPage && nextLower != null && catalog.PackageIdentity.Version < nextLower; + + if (catalog.PackageIdentity.Version <= pageInfo.Upper || hasGap) + { + itemIndex = await MergeEntryIntoPageAsync(context, catalog, pageIndex, itemIndex); + catalogIndex++; + } + else + { + // Before considering the current page "complete" ensure the page size is correct. + await EnsureValidPageSizeAtAsync(context, pageIndex); + + if (catalog.PackageIdentity.Version > pageInfo.Upper) + { + // The current catalog entry is greater than the current page's upper bound. This means we're done + // with the current page. Move to the next page and reset the item index. + pageIndex++; + itemIndex = 0; + } + } + } + + // Now that one of the two input lists is drained, handle the other (undrained) list. + + // Process the rest of the catalog leaves, if any. + if (catalogIndex < sortedCatalog.Count) + { + // Make sure there is at least one non-full page so that remaining catalog leaves can be pushed there. + if (pageInfos.Count == 0 + || pageInfos.Last().Count == MaxLeavesPerPage) + { + context.IndexInfo.Insert(pageIndex, PageInfo.New()); + } + else + { + pageIndex = pageInfos.Count - 1; + } + + // Push the remaining catalog leaves into the last page. + while (catalogIndex < sortedCatalog.Count) + { + var catalog = sortedCatalog[catalogIndex]; + itemIndex = await MergeEntryIntoPageAsync(context, catalog, pageIndex, itemIndex); + catalogIndex++; + } + + RemovePageAtIfEmpty(context, pageIndex); + } + + // Process the rest of the registration pages, if any, by ensuring the remaining pages are not too large. + while (pageIndex < pageInfos.Count) + { + await EnsureValidPageSizeAtAsync(context, pageIndex); + pageIndex++; + } + } + + private async Task MergeEntryIntoPageAsync( + Context context, + CatalogCommitItem entry, + int pageIndex, + int itemIndex) + { + var pageInfo = context.IndexInfo.Items[pageIndex]; + var items = await pageInfo.GetLeafInfosAsync(); + + while (itemIndex < items.Count) + { + if (entry.PackageIdentity.Version > items[itemIndex].Version) + { + // The current position in the item list is too low for the catalog version. Keep looking. + itemIndex++; + } + else if (entry.PackageIdentity.Version == items[itemIndex].Version) + { + if (entry.IsPackageDelete) + { + // Remove the registration leaf item. The current catalog commit item represents a + // delete for this version. + _logger.LogInformation( + "Version {Version} will be deleted by commit {CommitId}.", + entry.PackageIdentity.Version.ToNormalizedString(), + entry.CommitId); + context.DeletedLeaves.Add(await pageInfo.RemoveAtAsync(itemIndex)); + context.ModifiedPages.Add(pageInfo); + + RemovePageAtIfEmpty(context, pageIndex); + } + else + { + // Update the metadata of the existing registration leaf item. + _logger.LogInformation( + "Version {Version} will be updated by commit {CommitId}.", + entry.PackageIdentity.Version.ToNormalizedString(), + entry.CommitId); + context.ModifiedLeaves.Add(items[itemIndex]); + context.ModifiedPages.Add(pageInfo); + } + + // The version has been matched with an existing version. Leave the item index as-is. The next item + // is now at the current position. No more work is necessary. + return itemIndex; + } + else + { + break; + } + } + + await InsertAsync(context, pageInfo, itemIndex, entry); + + return itemIndex; + } + + private async Task InsertAsync( + Context context, + PageInfo pageInfo, + int index, + CatalogCommitItem entry) + { + if (entry.IsPackageDelete) + { + // No matching version was found for this delete. No more work is necessary. + _logger.LogInformation( + "Version {Version} does not exist. The delete from commit {CommitId} will have no affect.", + entry.PackageIdentity.Version.ToNormalizedString(), + entry.CommitId); + } + else + { + // Insert the new registration leaf item. + _logger.LogInformation( + "Version {Version} will be added by commit {CommitId}.", + entry.PackageIdentity.Version.ToNormalizedString(), + entry.CommitId); + var leafInfo = LeafInfo.New(entry.PackageIdentity.Version); + await pageInfo.InsertAsync(index, leafInfo); + context.ModifiedLeaves.Add(leafInfo); + context.ModifiedPages.Add(pageInfo); + } + } + + private async Task EnsureValidPageSizeAtAsync(Context context, int pageIndex) + { + var pageInfo = context.IndexInfo.Items[pageIndex]; + + // If we're not on the last page, pull items from the next page until the page is full or we've drained all + // of the subsequent pages and are now the last page. + while (pageInfo.Count < MaxLeavesPerPage && pageIndex < context.IndexInfo.Items.Count - 1) + { + var nextPageInfo = context.IndexInfo.Items[pageIndex + 1]; + var leafInfo = await nextPageInfo.RemoveAtAsync(0); + _logger.LogInformation( + "Page {PageNumber}/{PageCount} has too few items. Version {Version} will be moved from the next page.", + pageIndex + 1, + context.IndexInfo.Items.Count, + leafInfo.Version); + await pageInfo.InsertAsync(pageInfo.Count, leafInfo); + context.ModifiedPages.Add(pageInfo); + context.ModifiedPages.Add(nextPageInfo); + RemovePageAtIfEmpty(context, pageIndex + 1); + } + + // If the page is too large, push the extra items to the next page. + while (pageInfo.Count > MaxLeavesPerPage) + { + PageInfo nextPageInfo; + if (pageIndex == context.IndexInfo.Items.Count - 1) + { + nextPageInfo = PageInfo.New(); + context.IndexInfo.Insert(context.IndexInfo.Items.Count, nextPageInfo); + } + else + { + nextPageInfo = context.IndexInfo.Items[pageIndex + 1]; + } + + var leafInfo = await pageInfo.RemoveAtAsync(pageInfo.Count - 1); + _logger.LogInformation( + "Page {PageNumber}/{PageCount} has too many items. Version {Version} will be moved to the next page.", + pageIndex + 1, + context.IndexInfo.Items.Count, + leafInfo.Version); + await nextPageInfo.InsertAsync(0, leafInfo); + context.ModifiedPages.Add(pageInfo); + context.ModifiedPages.Add(nextPageInfo); + } + } + + private void RemovePageAtIfEmpty(Context context, int pageIndex) + { + var pageInfo = context.IndexInfo.Items[pageIndex]; + if (pageInfo.Count == 0) + { + _logger.LogInformation( + "The last version on page {PageNumber}/{PageCount} has been removed. The page itself will be removed.", + pageIndex + 1, + context.IndexInfo.Items.Count); + context.IndexInfo.RemoveAt(pageIndex); + context.ModifiedPages.Remove(pageInfo); + } + } + + private class Context + { + public Context(IndexInfo indexInfo, IReadOnlyList sortedCatalog) + { + SortedCatalog = sortedCatalog; + IndexInfo = indexInfo; + ModifiedPages = new HashSet(); + ModifiedLeaves = new HashSet(); + DeletedLeaves = new HashSet(); + } + + public IndexInfo IndexInfo { get; } + public IReadOnlyList SortedCatalog { get; } + public HashSet ModifiedPages { get; } + public HashSet ModifiedLeaves { get; } + public HashSet DeletedLeaves { get; } + } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Hives/HiveStorage.cs b/src/NuGet.Jobs.Catalog2Registration/Hives/HiveStorage.cs new file mode 100644 index 000000000..625acfcbe --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Hives/HiveStorage.cs @@ -0,0 +1,434 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Newtonsoft.Json; +using NuGet.Protocol; +using NuGet.Protocol.Catalog; +using NuGet.Protocol.Registration; +using NuGet.Versioning; +using NuGetGallery; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class HiveStorage : IHiveStorage + { + private readonly ICloudBlobClient _cloudBlobClient; + private readonly RegistrationUrlBuilder _urlBuilder; + private readonly IEntityBuilder _entityBuilder; + private readonly IThrottle _throttle; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public HiveStorage( + ICloudBlobClient cloudBlobClient, + RegistrationUrlBuilder urlBuilder, + IEntityBuilder entityBuilder, + IThrottle throttle, + IOptionsSnapshot options, + ILogger logger) + { + _cloudBlobClient = cloudBlobClient ?? throw new ArgumentNullException(nameof(cloudBlobClient)); + _urlBuilder = urlBuilder ?? throw new ArgumentNullException(nameof(urlBuilder)); + _entityBuilder = entityBuilder ?? throw new ArgumentNullException(nameof(entityBuilder)); + _throttle = throttle ?? throw new ArgumentNullException(nameof(throttle)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ReadIndexOrNullAsync(HiveType hive, string id) + { + var path = _urlBuilder.GetIndexPath(id); + return await ReadAsync(hive, path, "index", allow404: true); + } + + public async Task ReadPageAsync(HiveType hive, string url) + { + var path = _urlBuilder.ConvertToPath(hive, url); + return await ReadAsync(hive, path, "page", allow404: false); + } + + public async Task WriteIndexAsync( + HiveType hive, + IReadOnlyList replicaHives, + string id, + RegistrationIndex index) + { + var path = _urlBuilder.GetIndexPath(id); + await WriteAsync(hive, replicaHives, path, index, "index", _entityBuilder.UpdateIndexUrls); + } + + public async Task WritePageAsync( + HiveType hive, + IReadOnlyList replicaHives, + string id, + NuGetVersion lower, + NuGetVersion upper, + RegistrationPage page) + { + var path = _urlBuilder.GetPagePath(id, lower, upper); + await WriteAsync(hive, replicaHives, path, page, "page", _entityBuilder.UpdatePageUrls); + } + + public async Task WriteLeafAsync( + HiveType hive, + IReadOnlyList replicaHives, + string id, + NuGetVersion version, + RegistrationLeaf leaf) + { + var path = _urlBuilder.GetLeafPath(id, version); + await WriteAsync(hive, replicaHives, path, leaf, "leaf", _entityBuilder.UpdateLeafUrls); + } + + public async Task DeleteIndexAsync( + HiveType hive, + IReadOnlyList replicaHives, + string id) + { + var path = _urlBuilder.GetIndexPath(id); + await DeleteAsync(hive, replicaHives, path); + } + + public async Task DeleteUrlAsync( + HiveType hive, + IReadOnlyList replicaHives, + string url) + { + var path = _urlBuilder.ConvertToPath(hive, url); + await DeleteAsync(hive, replicaHives, path); + } + + private async Task ReadAsync( + HiveType hive, + string path, + string typeName, + bool allow404) + { + var blob = GetBlobReference(hive, path); + + _logger.LogInformation( + "Reading {TypeName} from container {Container} at path {Path}.", + typeName, + GetContainerName(hive), + path); + + await _throttle.WaitAsync(); + try + { + T result; + using (var blobStream = await blob.OpenReadAsync(AccessCondition.GenerateEmptyCondition())) + { + Stream readStream; + if (blob.Properties.ContentEncoding == "gzip") + { + readStream = new GZipStream(blobStream, CompressionMode.Decompress); + } + else + { + readStream = blobStream; + } + + using (readStream) + using (var streamReader = new StreamReader(readStream)) + using (var jsonTextReader = new JsonTextReader(streamReader)) + { + result = NuGetJsonSerialization.Serializer.Deserialize(jsonTextReader); + } + } + + _logger.LogInformation( + "Finished reading {TypeName} from container {Container} at path {Path} with Content-Encoding {ContentEncoding}.", + typeName, + GetContainerName(hive), + path, + blob.Properties.ContentEncoding); + + return result; + } + catch (StorageException ex) when (ex.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.NotFound) + { + _logger.LogInformation( + "No blob in container {Container} at path {Path} exists.", + GetContainerName(hive), + path, + blob.Properties.ContentEncoding); + + if (allow404) + { + return default(T); + } + else + { + throw; + } + } + finally + { + _throttle.Release(); + } + } + + private MemoryStream Serialize(HiveType hive, T entity) + { + var serializedStream = new MemoryStream(); + Stream writeStream; + GZipStream gzipStream; + if (IsGzipped(hive)) + { + gzipStream = new GZipStream( + serializedStream, + CompressionMode.Compress, + leaveOpen: true); + writeStream = gzipStream; + } + else + { + gzipStream = null; + writeStream = serializedStream; + } + + using (var streamWriter = new StreamWriter( + writeStream, + new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), + bufferSize: 1024, + leaveOpen: true)) + using (var jsonTextWriter = new JsonTextWriter(streamWriter)) + { + NuGetJsonSerialization.Serializer.Serialize(jsonTextWriter, entity); + } + + gzipStream?.Dispose(); + serializedStream.Position = 0; + return serializedStream; + } + + private async Task WriteAsync( + HiveType hive, + IReadOnlyList replicaHives, + string path, + T entity, + string typeName, + Action updateUrls) + { + var hiveToWriteTask = new Dictionary(); + var previousHive = hive; + foreach (var currentHive in GetHiveSequence(hive, replicaHives)) + { + if (hiveToWriteTask.ContainsKey(currentHive)) + { + continue; + } + + if (currentHive != previousHive) + { + // The hive defines the base URL. If we're now writing to another hive (which is the case if this + // is the 2nd ... Nth hive) then we need to update the URLs to point to the current hive. + updateUrls(entity, previousHive, currentHive); + } + + // Serialize in sequence since we have a single entity that needs to be written to multiple hives and + // therefore have multiple URL base addresses for the various serializations. + var memoryStream = Serialize(currentHive, entity); + + // Write in parallel since we've captured the serialized bytes at this point and can safely proceed to + // the next hive which will update the entity URLs. + var writeTask = WriteAsync(currentHive, path, memoryStream, typeName); + hiveToWriteTask.Add(currentHive, writeTask); + + previousHive = currentHive; + } + + // Return the URLs to their original state so that the caller can consider replica hives as an + // implementation detail and not need to worry about the entity changing during the process. + if (previousHive != hive) + { + updateUrls(entity, previousHive, hive); + } + + await Task.WhenAll(hiveToWriteTask.Values); + } + + private static IEnumerable GetHiveSequence(HiveType hive, IReadOnlyList replicaHives) + { + yield return hive; + foreach (var replicaHive in replicaHives) + { + yield return replicaHive; + } + } + + private async Task WriteAsync( + HiveType hive, + string path, + MemoryStream memoryStream, + string typeName) + { + using (memoryStream) + { + var container = GetContainer(hive); + var blob = GetBlobReference(container, path); + + blob.Properties.ContentType = "application/json"; + blob.Properties.CacheControl = "no-store"; + + if (IsGzipped(hive)) + { + blob.Properties.ContentEncoding = "gzip"; + } + + _logger.LogInformation( + "Writing {TypeName} ({ByteCount} bytes) to container {Container} at path {Path} with Content-Encoding {ContentEncoding}.", + typeName, + memoryStream.Length, + GetContainerName(hive), + path, + blob.Properties.ContentEncoding); + + await _throttle.WaitAsync(); + try + { + await blob.UploadFromStreamAsync(memoryStream, AccessCondition.GenerateEmptyCondition()); + + if (_options.Value.EnsureSingleSnapshot) + { + var segment = await container.ListBlobsSegmentedAsync( + path, + useFlatBlobListing: true, + blobListingDetails: BlobListingDetails.Snapshots, + maxResults: 2, + blobContinuationToken: null, + options: null, + operationContext: null, + cancellationToken: CancellationToken.None); + + if (segment.Results.Count == 1) + { + _logger.LogInformation( + "The {TypeName} blob in container {Container} at path {Path} does not have a snapshot so one will be created.", + typeName, + GetContainerName(hive), + path); + await blob.SnapshotAsync(CancellationToken.None); + } + else + { + _logger.LogInformation( + "The {TypeName} blob in container {Container} at path {Path} already has a snapshot.", + typeName, + GetContainerName(hive), + path); + } + } + } + finally + { + _throttle.Release(); + } + + _logger.LogInformation( + "Finished writing {TypeName} to container {Container} and path {Path}.", + typeName, + GetContainerName(hive), + path); + } + } + + private async Task DeleteAsync(HiveType hive, IReadOnlyList replicaHives, string path) + { + var deleteTasks = GetHiveSequence(hive, replicaHives) + .Select(x => DeleteAsync(x, path)) + .ToList(); + await Task.WhenAll(deleteTasks); + } + + private async Task DeleteAsync(HiveType hive, string path) + { + var blob = GetBlobReference(hive, path); + + await _throttle.WaitAsync(); + try + { + if (await blob.ExistsAsync()) + { + _logger.LogInformation( + "Deleting blob in container {Container} at path {Path}.", + GetContainerName(hive), + path); + await blob.DeleteIfExistsAsync(); + _logger.LogInformation( + "Finished deleting blob in container {Container} at path {Path}.", + GetContainerName(hive), + path); + } + else + { + _logger.LogInformation( + "No blob in container {Container} at path {Path} exists so no delete is required.", + GetContainerName(hive), + path); + } + } + finally + { + _throttle.Release(); + } + } + + private ISimpleCloudBlob GetBlobReference(ICloudBlobContainer container, string path) + { + var blob = container.GetBlobReference(Uri.UnescapeDataString(path)); + return blob; + } + + private ISimpleCloudBlob GetBlobReference(HiveType hive, string path) + { + var container = GetContainer(hive); + return GetBlobReference(container, path); + } + + private bool IsGzipped(HiveType hive) + { + return hive != HiveType.Legacy; + } + + private string GetContainerName(HiveType hive) + { + string container; + switch (hive) + { + case HiveType.Legacy: + container = _options.Value.LegacyStorageContainer; + break; + case HiveType.Gzipped: + container = _options.Value.GzippedStorageContainer; + break; + case HiveType.SemVer2: + container = _options.Value.SemVer2StorageContainer; + break; + default: + throw new NotImplementedException($"The hive type '{hive}' does not have configured storage container."); + } + + return container; + } + + private ICloudBlobContainer GetContainer(HiveType hive) + { + var containerName = GetContainerName(hive); + return _cloudBlobClient.GetContainerReference(containerName); + } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Hives/HiveType.cs b/src/NuGet.Jobs.Catalog2Registration/Hives/HiveType.cs new file mode 100644 index 000000000..014d85eba --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Hives/HiveType.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Jobs.Catalog2Registration +{ + /// + /// This represents the distinct hive versions released. See the following documentation for more details: + /// https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#versioning + /// + public enum HiveType + { + /// + /// Non-gzipped blobs. Only SemVer 1.0.0 packages. This is denoted by service index type "RegistrationsBaseUrl". + /// + Legacy, + + /// + /// Gzipped blobs. Only SemVer 1.0.0 packages. This is denoted by service index type "RegistrationsBaseUrl/3.4.0". + /// + Gzipped, + + /// + /// Gzipped blobs. SemVer 1.0.0 and SemVer 2.0.0 packages. This is denoted by service index type "RegistrationsBaseUrl/3.6.0". + /// + SemVer2, + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Hives/HiveUpdater.cs b/src/NuGet.Jobs.Catalog2Registration/Hives/HiveUpdater.cs new file mode 100644 index 000000000..57fab3ae8 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Hives/HiveUpdater.cs @@ -0,0 +1,498 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Protocol.Catalog; +using NuGet.Services; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Versioning; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class HiveUpdater : IHiveUpdater + { + private static readonly IReadOnlyList DeleteUris = new[] { Schema.DataTypes.PackageDelete }; + + private readonly IHiveStorage _storage; + private readonly IHiveMerger _merger; + private readonly IEntityBuilder _entityBuilder; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public HiveUpdater( + IHiveStorage storage, + IHiveMerger merger, + IEntityBuilder entityBuilder, + IOptionsSnapshot options, + ILogger logger) + { + _storage = storage ?? throw new ArgumentNullException(nameof(storage)); + _merger = merger ?? throw new ArgumentNullException(nameof(merger)); + _entityBuilder = entityBuilder ?? throw new ArgumentNullException(nameof(entityBuilder)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task UpdateAsync( + HiveType hive, + IReadOnlyList replicaHives, + string id, + IReadOnlyList entries, + IReadOnlyDictionary entryToCatalogLeaf, + CatalogCommit registrationCommit) + { + // Validate the input and put it in more convenient forms. + if (!entries.Any()) + { + return; + } + GuardInput(entries, entryToCatalogLeaf); + var sortedCatalog = entries.OrderBy(x => x.PackageIdentity.Version).ToList(); + var versionToCatalogLeaf = entryToCatalogLeaf.ToDictionary(x => x.Key.PackageIdentity.Version, x => x.Value); + + // Remove SemVer 2.0.0 versions if this hive should only have SemVer 1.0.0 versions. + if (ShouldExcludeSemVer2(hive)) + { + Guard.Assert( + replicaHives.All(ShouldExcludeSemVer2), + "A replica hive of a non-SemVer 2.0.0 hive must also exclude SemVer 2.0.0."); + + ExcludeSemVer2(hive, sortedCatalog, versionToCatalogLeaf); + } + else + { + Guard.Assert( + replicaHives.All(x => !ShouldExcludeSemVer2(x)), + "A replica hive of a SemVer 2.0.0 hive must also include SemVer 2.0.0."); + } + + _logger.LogInformation( + "Starting to update the {PackageId} registration index in the {Hive} hive and {ReplicaHives} replica hives with {UpsertCount} " + + "package details and {DeleteCount} package deletes.", + id, + hive, + replicaHives, + entryToCatalogLeaf.Count, + entries.Count - entryToCatalogLeaf.Count); + + // Read the existing registration index if it exists. If it does not exist, initialize a new index. + var index = await _storage.ReadIndexOrNullAsync(hive, id); + IndexInfo indexInfo; + if (index == null) + { + indexInfo = IndexInfo.New(); + } + else + { + indexInfo = IndexInfo.Existing(_storage, hive, index); + } + + // Find all of the existing page URLs. This will be used later to find orphan pages. + var existingPageUrls = GetPageUrls(indexInfo); + + // Read all of the obviously relevant pages in parallel. This simply evaluates some work that would + // otherwise be done lazily. + await LoadRelevantPagesAsync(sortedCatalog, indexInfo); + + // Merge the incoming catalog entries in memory. + var mergeResult = await _merger.MergeAsync(indexInfo, sortedCatalog); + + // Write the modified leaves. + await UpdateLeavesAsync(hive, replicaHives, id, versionToCatalogLeaf, registrationCommit, mergeResult); + + // Write the pages and handle the inline vs. non-inlined cases. + if (indexInfo.Items.Count == 0) + { + _logger.LogInformation("There are no pages to update."); + } + else + { + var itemCount = indexInfo.Items.Sum(x => x.Count); + if (itemCount <= _options.Value.MaxInlinedLeafItems) + { + _logger.LogInformation( + "There are {Count} total leaf items so the leaf items will be inlined.", + itemCount); + + await UpdateInlinedPagesAsync(hive, id, indexInfo, registrationCommit); + } + else + { + _logger.LogInformation( + "There are {Count} total leaf items so the leaf items will not be inlined.", + itemCount); + + await UpdateNonInlinedPagesAsync(hive, replicaHives, id, indexInfo, registrationCommit, mergeResult); + } + } + + // Write the index, if there were any changes. + if (mergeResult.ModifiedPages.Any() || mergeResult.ModifiedLeaves.Any()) + { + _logger.LogInformation("Updating the index."); + _entityBuilder.UpdateIndex(indexInfo.Index, hive, id, indexInfo.Items.Count); + _entityBuilder.UpdateCommit(indexInfo.Index, registrationCommit); + await _storage.WriteIndexAsync(hive, replicaHives, id, indexInfo.Index); + } + + if (!indexInfo.Items.Any()) + { + _logger.LogInformation("Deleting the index since there are no more page items."); + await _storage.DeleteIndexAsync(hive, replicaHives, id); + } + + // Delete orphan blobs. + await DeleteOrphansAsync(hive, replicaHives, existingPageUrls, indexInfo, mergeResult); + + _logger.LogInformation( + "Done updating the {PackageId} registration index in the {Hive} hive and replica hives {ReplicaHives}. {ModifiedPages} pages were " + + "updated, {ModifiedLeaves} leaves were upserted, and {DeletedLeaves} leaves were deleted.", + id, + hive, + replicaHives, + mergeResult.ModifiedPages.Count, + mergeResult.ModifiedLeaves.Count, + mergeResult.DeletedLeaves.Count); + } + + private static bool ShouldExcludeSemVer2(HiveType hive) + { + return hive == HiveType.Legacy || hive == HiveType.Gzipped; + } + + private void ExcludeSemVer2( + HiveType hive, + List sortedCatalog, + Dictionary versionToCatalogLeaf) + { + Guard.Assert( + hive == HiveType.Legacy || hive == HiveType.Gzipped, + "Only the legacy and gzipped hives should exclude SemVer 2.0.0 versions."); + + for (int i = 0; i < sortedCatalog.Count; i++) + { + var catalogCommitItem = sortedCatalog[i]; + if (catalogCommitItem.IsPackageDelete) + { + continue; + } + + var version = catalogCommitItem.PackageIdentity.Version; + var catalogLeaf = versionToCatalogLeaf[version]; + if (catalogLeaf.IsSemVer2()) + { + // Turn the PackageDetails into a PackageDelete to ensure that a known SemVer 2.0.0 version is not + // in a hive that should not have SemVer 2.0.0. This may cause a little bit more work (like a + // non-inlined page getting downloaded) but it's worth it to allow reflows of SemVer 2.0.0 packages + // to fix up problems. In general, reflow should be a powerful fix-up tool. If this causes + // performance issues later, we can revisit this extra work at that time. + sortedCatalog[i] = new CatalogCommitItem( + catalogCommitItem.Uri, + catalogCommitItem.CommitId, + catalogCommitItem.CommitTimeStamp, + types: null, + typeUris: DeleteUris, + packageIdentity: catalogCommitItem.PackageIdentity); + versionToCatalogLeaf.Remove(version); + + _logger.LogInformation( + "Version {Version} is SemVer 2.0.0 so it will be treated as a package delete.", + catalogLeaf.ParsePackageVersion().ToFullString(), + hive); + } + } + } + + private async Task LoadRelevantPagesAsync(List sortedCatalog, IndexInfo indexInfo) + { + // If there are no page items at all, there is no work to do. + if (indexInfo.Items.Count == 0) + { + return; + } + + var catalogIndex = 0; + var pageIndex = 0; + var relevantPages = new ConcurrentBag(); + + // Load pages where at least one catalog item falls in the bounds of the page. + while (catalogIndex < sortedCatalog.Count && pageIndex < indexInfo.Items.Count) + { + var currentCatalog = sortedCatalog[catalogIndex]; + var currentPage = indexInfo.Items[pageIndex]; + + if (currentCatalog.PackageIdentity.Version < currentPage.Lower) + { + // The current catalog item lower than the current page's bounds. Move on to the next catalog item. + catalogIndex++; + } + else if (currentCatalog.PackageIdentity.Version <= currentPage.Upper) + { + // The current catalog item is inside the current page's bounds. This page should be downloaded. + if (!currentPage.IsInlined) + { + _logger.LogInformation( + "Preemptively loading page {PageNumber}/{PageCount} [{Lower}, {Upper}] since catalog version {Version} is in its bounds.", + pageIndex + 1, + indexInfo.Items.Count, + currentPage.Lower.ToNormalizedString(), + currentPage.Upper.ToNormalizedString(), + currentCatalog.PackageIdentity.Version.ToNormalizedString()); + + relevantPages.Add(currentPage); + } + + // This page is now included in the set of relevant pages. No need to consider it any more. + pageIndex++; + } + else + { + // The current catalog item is higher than the current page's bounds. This page is not relevant. + pageIndex++; + } + } + + await ParallelAsync.Repeat( + async () => + { + await Task.Yield(); + while (relevantPages.TryTake(out var pageInfo)) + { + await pageInfo.GetLeafInfosAsync(); + } + }, + _options.Value.MaxConcurrentOperationsPerHive); + } + + private async Task UpdateLeavesAsync( + HiveType hive, + IReadOnlyList replicaHives, + string id, + Dictionary versionToCatalogLeaf, + CatalogCommit registrationCommit, + HiveMergeResult mergeResult) + { + if (!mergeResult.ModifiedLeaves.Any()) + { + _logger.LogInformation("No leaves need to be updated."); + return; + } + + _logger.LogInformation( + "Updating {Count} registration leaves.", + mergeResult.ModifiedLeaves.Count, + id, + hive); + + var taskFactories = new ConcurrentBag>(); + foreach (var leafInfo in mergeResult.ModifiedLeaves) + { + _entityBuilder.UpdateLeafItem(leafInfo.LeafItem, hive, id, versionToCatalogLeaf[leafInfo.Version]); + _entityBuilder.UpdateCommit(leafInfo.LeafItem, registrationCommit); + var leaf = _entityBuilder.NewLeaf(leafInfo.LeafItem); + taskFactories.Add(async () => + { + _logger.LogInformation("Updating leaf {PackageId} {Version}.", id, leafInfo.Version.ToNormalizedString()); + await _storage.WriteLeafAsync(hive, replicaHives, id, leafInfo.Version, leaf); + }); + } + + await ParallelAsync.Repeat( + async () => + { + await Task.Yield(); + while (taskFactories.TryTake(out var taskFactory)) + { + await taskFactory(); + } + }, + _options.Value.MaxConcurrentOperationsPerHive); + } + + private async Task UpdateInlinedPagesAsync( + HiveType hive, + string id, + IndexInfo indexInfo, + CatalogCommit registrationCommit) + { + for (var pageIndex = 0; pageIndex < indexInfo.Items.Count; pageIndex++) + { + var pageInfo = indexInfo.Items[pageIndex]; + + if (!pageInfo.IsInlined) + { + _logger.LogInformation( + "Moving page {PageNumber}/{PageCount} [{Lower}, {Upper}] from having its own blob to being inlined.", + pageIndex + 1, + indexInfo.Items.Count, + pageInfo.Lower.ToNormalizedString(), + pageInfo.Upper.ToNormalizedString()); + + pageInfo = await pageInfo.CloneToInlinedAsync(); + indexInfo.RemoveAt(pageIndex); + indexInfo.Insert(pageIndex, pageInfo); + } + + Guard.Assert(pageInfo.IsInlined, "The page should be inlined at this point."); + + _entityBuilder.UpdateInlinedPageItem(pageInfo.PageItem, hive, id, pageInfo.Count, pageInfo.Lower, pageInfo.Upper); + _entityBuilder.UpdateCommit(pageInfo.PageItem, registrationCommit); + } + } + + private async Task UpdateNonInlinedPagesAsync( + HiveType hive, + IReadOnlyList replicaHives, + string id, + IndexInfo indexInfo, + CatalogCommit registrationCommit, + HiveMergeResult mergeResult) + { + var taskFactories = new ConcurrentBag>(); + for (var pageIndex = 0; pageIndex < indexInfo.Items.Count; pageIndex++) + { + var pageInfo = indexInfo.Items[pageIndex]; + + if (pageInfo.IsInlined) + { + _logger.LogInformation( + "Moving page {PageNumber}/{PageCount} [{Lower}, {Upper}] from being inlined to having its own blob.", + pageIndex + 1, + indexInfo.Items.Count, + pageInfo.Lower.ToNormalizedString(), + pageInfo.Upper.ToNormalizedString()); + + pageInfo = await pageInfo.CloneToNonInlinedAsync(); + indexInfo.RemoveAt(pageIndex); + indexInfo.Insert(pageIndex, pageInfo); + } + else if (!mergeResult.ModifiedPages.Contains(pageInfo)) + { + _logger.LogInformation( + "Skipping unmodified page {PageNumber}/{PageCount} [{Lower}, {Upper}].", + pageIndex + 1, + indexInfo.Items.Count, + pageInfo.Lower.ToNormalizedString(), + pageInfo.Upper.ToNormalizedString()); + + continue; + } + + Guard.Assert(!pageInfo.IsInlined, "The page should not be inlined at this point."); + + var page = await pageInfo.GetPageAsync(); + _entityBuilder.UpdateNonInlinedPageItem(pageInfo.PageItem, hive, id, pageInfo.Count, pageInfo.Lower, pageInfo.Upper); + _entityBuilder.UpdateCommit(pageInfo.PageItem, registrationCommit); + _entityBuilder.UpdatePage(page, hive, id, pageInfo.Count, pageInfo.Lower, pageInfo.Upper); + _entityBuilder.UpdateCommit(page, registrationCommit); + + var pageNumber = pageIndex + 1; + taskFactories.Add(async () => + { + _logger.LogInformation( + "Updating page {PageNumber}/{PageCount} [{Lower}, {Upper}].", + pageNumber, + indexInfo.Items.Count, + pageInfo.Lower.ToNormalizedString(), + pageInfo.Upper.ToNormalizedString()); + await _storage.WritePageAsync(hive, replicaHives, id, pageInfo.Lower, pageInfo.Upper, page); + }); + } + + await ParallelAsync.Repeat( + async () => + { + await Task.Yield(); + while (taskFactories.TryTake(out var taskFactory)) + { + await taskFactory(); + } + }, + _options.Value.MaxConcurrentOperationsPerHive); + } + + private static void GuardInput( + IReadOnlyList entries, + IReadOnlyDictionary entryToCatalogLeaf) + { + var uniqueVersions = new HashSet(); + foreach (var entry in entries) + { + Guard.Assert( + entry.IsPackageDelete ^ entry.IsPackageDetails, + "A catalog commit item must be either a PackageDelete or a package details but not both."); + Guard.Assert( + uniqueVersions.Add(entry.PackageIdentity.Version), + "There must be exactly on catalog commit item per version."); + + if (entry.IsPackageDetails) + { + Guard.Assert( + entryToCatalogLeaf.ContainsKey(entry), + "Each PackageDetails catalog commit item must have an associate catalog leaf."); + } + } + } + + private HashSet GetPageUrls(IndexInfo indexInfo) + { + return new HashSet(indexInfo + .Items + .Where(x => !x.IsInlined) + .Select(x => x.PageItem.Url)); + } + + private async Task DeleteOrphansAsync( + HiveType hive, + IReadOnlyList replicaHives, + IEnumerable existingPageUrls, + IndexInfo indexInfo, + HiveMergeResult mergeResult) + { + // Start with all of the page URLs found in the index prior to the update process. + var orphanUrls = new HashSet(existingPageUrls); + + // Add all of the deleted leaf URLs. + orphanUrls.UnionWith(mergeResult.DeletedLeaves.Select(x => x.LeafItem.Url)); + + // Leave the new page URLs alone. + foreach (var pageInfo in indexInfo.Items) + { + orphanUrls.Remove(pageInfo.PageItem.Url); + } + + // Leave the modified leaf URLs alone. This should not be necessary since deleted leaves and modified + // leaves are disjoint sets but is a reasonable precaution. + foreach (var leafInfo in mergeResult.ModifiedLeaves) + { + orphanUrls.Remove(leafInfo.LeafItem.Url); + } + + if (orphanUrls.Count == 0) + { + _logger.LogInformation("There are no orphan blobs to delete."); + return; + } + + _logger.LogInformation("About to delete {Count} orphan blobs.", orphanUrls.Count); + var work = new ConcurrentBag(orphanUrls); + await ParallelAsync.Repeat( + async () => + { + while (work.TryTake(out var url)) + { + await _storage.DeleteUrlAsync(hive, replicaHives, url); + } + }, + _options.Value.MaxConcurrentOperationsPerHive); + _logger.LogInformation("Done deleting orphan blobs.", orphanUrls.Count); + } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Hives/IHiveMerger.cs b/src/NuGet.Jobs.Catalog2Registration/Hives/IHiveMerger.cs new file mode 100644 index 000000000..6a1ef2af4 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Hives/IHiveMerger.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog; + +namespace NuGet.Jobs.Catalog2Registration +{ + public interface IHiveMerger + { + /// + /// Merge the incoming list of catalog items into the registration index. This method will not go into the + /// details of updating the leaf item metadata. That responsiblity is left up to the caller who can inspect the + /// . This logic also does not handle the inlining or externalizing + /// of leaf items since this is more of a storage concern. The provided bookkeeping + /// object and its child object will be modified and the changes will be detailed in the returned + /// . + /// + Task MergeAsync(IndexInfo indexInfo, IReadOnlyList sortedCatalog); + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Hives/IHiveStorage.cs b/src/NuGet.Jobs.Catalog2Registration/Hives/IHiveStorage.cs new file mode 100644 index 000000000..a61f792f5 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Hives/IHiveStorage.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGet.Protocol.Registration; +using NuGet.Versioning; + +namespace NuGet.Jobs.Catalog2Registration +{ + public interface IHiveStorage + { + Task DeleteIndexAsync(HiveType hive, IReadOnlyList replicaHives, string id); + Task DeleteUrlAsync(HiveType hive, IReadOnlyList replicaHives, string url); + Task ReadIndexOrNullAsync(HiveType hive, string id); + Task ReadPageAsync(HiveType hive, string url); + Task WriteIndexAsync(HiveType hive, IReadOnlyList replicaHives, string id, RegistrationIndex index); + Task WriteLeafAsync(HiveType hive, IReadOnlyList replicaHives, string id, NuGetVersion version, RegistrationLeaf leaf); + Task WritePageAsync(HiveType hive, IReadOnlyList replicaHives, string id, NuGetVersion lower, NuGetVersion upper, RegistrationPage page); + } +} \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2Registration/Hives/IHiveUpdater.cs b/src/NuGet.Jobs.Catalog2Registration/Hives/IHiveUpdater.cs new file mode 100644 index 000000000..e25c8fb8f --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Hives/IHiveUpdater.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGet.Protocol.Catalog; +using NuGet.Services.Metadata.Catalog; + +namespace NuGet.Jobs.Catalog2Registration +{ + public interface IHiveUpdater + { + Task UpdateAsync( + HiveType hive, + IReadOnlyList replicaHives, + string id, + IReadOnlyList entries, + IReadOnlyDictionary entryToLeaf, + CatalogCommit registrationCommit); + } +} \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2Registration/IRegistrationUpdater.cs b/src/NuGet.Jobs.Catalog2Registration/IRegistrationUpdater.cs new file mode 100644 index 000000000..8de3afd4f --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/IRegistrationUpdater.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGet.Protocol.Catalog; +using NuGet.Services.Metadata.Catalog; + +namespace NuGet.Jobs.Catalog2Registration +{ + public interface IRegistrationUpdater + { + Task UpdateAsync( + string id, + IReadOnlyList entries, + IReadOnlyDictionary entryToLeaf); + } +} \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2Registration/Job.cs b/src/NuGet.Jobs.Catalog2Registration/Job.cs new file mode 100644 index 000000000..1780d7ef7 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Job.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Net; +using System.Threading.Tasks; +using Autofac; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using NuGet.Jobs.Catalog2Registration; +using NuGet.Services.V3; + +namespace NuGet.Jobs +{ + public class Job : JsonConfigurationJob + { + private const string ConfigurationSectionName = "Catalog2Registration"; + + public override async Task Run() + { + ServicePointManager.DefaultConnectionLimit = 64; + ServicePointManager.MaxServicePointIdleTime = 10000; + + await _serviceProvider.GetRequiredService().ExecuteAsync(); + } + + protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder, IConfigurationRoot configurationRoot) + { + containerBuilder.AddCatalog2Registration(); + } + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + services.AddCatalog2Registration(GlobalTelemetryDimensions); + + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/NuGet.Jobs.Catalog2Registration.csproj b/src/NuGet.Jobs.Catalog2Registration/NuGet.Jobs.Catalog2Registration.csproj new file mode 100644 index 000000000..8f785559e --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/NuGet.Jobs.Catalog2Registration.csproj @@ -0,0 +1,120 @@ + + + + + + Debug + AnyCPU + {5ABE8807-2209-4948-9FC5-1980A507C47A} + Exe + NuGet.Jobs.Catalog2Registration + NuGet.Jobs.Catalog2Registration + v4.7.2 + 512 + true + true + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + false + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0.3.0 + runtime; build; native; contentfiles; analyzers + all + + + + + {E97F23B8-ECB0-4AFA-B00C-015C39395FEF} + NuGet.Services.Metadata.Catalog + + + {4b4b1efb-8f33-42e6-b79f-54e7f3293d31} + NuGet.Jobs.Common + + + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} + NuGet.Protocol.Catalog + + + {c3f9a738-9759-4b2b-a50d-6507b28a659b} + NuGet.Services.V3 + + + + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + + + \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2Registration/NuGet.Jobs.Catalog2Registration.nuspec b/src/NuGet.Jobs.Catalog2Registration/NuGet.Jobs.Catalog2Registration.nuspec new file mode 100644 index 000000000..5f3ced6f5 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/NuGet.Jobs.Catalog2Registration.nuspec @@ -0,0 +1,16 @@ + + + + Catalog2Registration.Rebuild + $version$ + .NET Foundation + .NET Foundation + Catalog2Registration + Copyright .NET Foundation + + + + + + + \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2Registration/Program.cs b/src/NuGet.Jobs.Catalog2Registration/Program.cs new file mode 100644 index 000000000..30141a7ca --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Program.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Jobs.Catalog2Registration +{ + class Program + { + static int Main(string[] args) + { + var job = new Job(); + return JobRunner.Run(job, args).GetAwaiter().GetResult(); + } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Properties/AssemblyInfo.cs b/src/NuGet.Jobs.Catalog2Registration/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..d3c3a5a65 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Jobs.Catalog2Registration")] +[assembly: ComVisible(false)] +[assembly: Guid("5abe8807-2209-4948-9fc5-1980a507c47a")] diff --git a/src/NuGet.Jobs.Catalog2Registration/RegistrationCollectorLogic.cs b/src/NuGet.Jobs.Catalog2Registration/RegistrationCollectorLogic.cs new file mode 100644 index 000000000..48b7c5b57 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/RegistrationCollectorLogic.cs @@ -0,0 +1,84 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Services; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.V3; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class RegistrationCollectorLogic : ICommitCollectorLogic + { + private readonly CommitCollectorUtility _utility; + private readonly IRegistrationUpdater _updater; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public RegistrationCollectorLogic( + CommitCollectorUtility utility, + IRegistrationUpdater updater, + IOptionsSnapshot options, + ILogger logger) + { + _utility = utility ?? throw new ArgumentNullException(nameof(utility)); + _updater = updater ?? throw new ArgumentNullException(nameof(updater)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (_options.Value.MaxConcurrentIds <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(options), + $"The {nameof(Catalog2RegistrationConfiguration.MaxConcurrentIds)} must be greater than zero."); + } + } + + public Task> CreateBatchesAsync(IEnumerable catalogItems) + { + // Create a single batch of all unprocessed catalog items so that we can have complete control of the + // parallelism in this class. + return Task.FromResult(_utility.CreateSingleBatch(catalogItems)); + } + + public async Task OnProcessBatchAsync(IEnumerable items) + { + var itemList = items.ToList(); + _logger.LogInformation("Got {Count} catalog commit items to process.", itemList.Count); + + var latestItems = _utility.GetLatestPerIdentity(itemList); + _logger.LogInformation("Got {Count} unique package identities.", latestItems.Count); + + var allWork = _utility.GroupById(latestItems); + _logger.LogInformation("Got {Count} unique IDs.", allWork.Count); + + var allEntryToLeaf = await _utility.GetEntryToDetailsLeafAsync(latestItems); + _logger.LogInformation("Fetched {Count} package details leaves.", allEntryToLeaf.Count); + + _logger.LogInformation("Starting {Count} workers processing each package ID batch.", _options.Value.MaxConcurrentIds); + await ParallelAsync.Repeat( + async () => + { + await Task.Yield(); + while (allWork.TryTake(out var work)) + { + var entryToLeaf = work + .Value + .Where(CommitCollectorUtility.IsOnlyPackageDetails) + .ToDictionary(e => e, e => allEntryToLeaf[e], ReferenceEqualityComparer.Default); + + await _updater.UpdateAsync(work.Id, work.Value, entryToLeaf); + } + }, + _options.Value.MaxConcurrentIds); + + _logger.LogInformation("All workers have completed successfully."); + } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/RegistrationUpdater.cs b/src/NuGet.Jobs.Catalog2Registration/RegistrationUpdater.cs new file mode 100644 index 000000000..154209f86 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/RegistrationUpdater.cs @@ -0,0 +1,96 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Protocol.Catalog; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class RegistrationUpdater : IRegistrationUpdater + { + private static readonly Dictionary> HiveToReplicaHives = new Dictionary> + { + { + HiveType.Legacy, + new List { HiveType.Gzipped } + }, + { + HiveType.SemVer2, + new List() + }, + }; + + private readonly IHiveUpdater _hiveUpdater; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public RegistrationUpdater( + IHiveUpdater hiveUpdater, + IOptionsSnapshot options, + ILogger logger) + { + _hiveUpdater = hiveUpdater ?? throw new ArgumentNullException(nameof(hiveUpdater)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (_options.Value.MaxConcurrentHivesPerId <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(options), + $"The {nameof(Catalog2RegistrationConfiguration.MaxConcurrentHivesPerId)} must be greater than zero."); + } + } + + public async Task UpdateAsync( + string id, + IReadOnlyList entries, + IReadOnlyDictionary entryToLeaf) + { + var registrationCommitTimestamp = DateTimeOffset.UtcNow; + var hives = new ConcurrentBag(HiveToReplicaHives.Keys); + await ParallelAsync.Repeat( + async () => + { + await Task.Yield(); + while (hives.TryTake(out var hive)) + { + await ProcessHiveAsync(hive, id, entries, entryToLeaf, registrationCommitTimestamp); + } + }, + _options.Value.MaxConcurrentHivesPerId); + } + + private async Task ProcessHiveAsync( + HiveType hive, + string id, + IReadOnlyList entries, + IReadOnlyDictionary entryToLeaf, + DateTimeOffset registrationCommitTimestamp) + { + var registrationCommitId = Guid.NewGuid().ToString(); + var registrationCommit = new CatalogCommit(registrationCommitId, registrationCommitTimestamp); + using (_logger.BeginScope( + "Processing package {PackageId}, " + + "hive {Hive}, " + + "replica hives {ReplicaHives}, " + + "registration commit ID {RegistrationCommitId}, " + + "registration commit timestamp {RegistrationCommitTimestamp:O}.", + id, + hive, + HiveToReplicaHives[hive], + registrationCommitId, + registrationCommitTimestamp)) + { + _logger.LogInformation("Processing {Count} catalog commit items.", entries.Count); + await _hiveUpdater.UpdateAsync(hive, HiveToReplicaHives[hive], id, entries, entryToLeaf, registrationCommit); + } + } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Schema/EntityBuilder.cs b/src/NuGet.Jobs.Catalog2Registration/Schema/EntityBuilder.cs new file mode 100644 index 000000000..4b9367e3f --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Schema/EntityBuilder.cs @@ -0,0 +1,328 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Options; +using NuGet.Protocol.Catalog; +using NuGet.Protocol.Registration; +using NuGet.Services; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class EntityBuilder : IEntityBuilder + { + private readonly RegistrationUrlBuilder _urlBuilder; + private readonly IOptionsSnapshot _options; + private readonly FlatContainerPackagePathProvider _flatContainerPathProvider; + private readonly Uri _galleryBaseUrl; + + public EntityBuilder( + RegistrationUrlBuilder urlBuilder, + IOptionsSnapshot options) + { + _urlBuilder = urlBuilder ?? throw new ArgumentNullException(nameof(urlBuilder)); + _options = options ?? throw new ArgumentNullException(nameof(urlBuilder)); + + var flatContainerBaseUrl = _options.Value.FlatContainerBaseUrl.TrimEnd('/'); + _flatContainerPathProvider = new FlatContainerPackagePathProvider(flatContainerBaseUrl); + _galleryBaseUrl = new Uri(_options.Value.GalleryBaseUrl); + } + + public void UpdateLeafItem(RegistrationLeafItem leafItem, HiveType hive, string id, PackageDetailsCatalogLeaf packageDetails) + { + var parsedVersion = packageDetails.ParsePackageVersion(); + + leafItem.Url = _urlBuilder.GetLeafUrl(hive, id, parsedVersion); + leafItem.Type = JsonLdConstants.RegistrationLeafItemType; + leafItem.PackageContent = GetPackageContentUrl(id, packageDetails); + leafItem.Registration = _urlBuilder.GetIndexUrl(hive, id); + + if (leafItem.CatalogEntry == null) + { + leafItem.CatalogEntry = new Protocol.Registration.RegistrationCatalogEntry(); + } + + UpdateCatalogEntry(hive, id, leafItem.CatalogEntry, packageDetails, parsedVersion); + } + + private void UpdateCatalogEntry( + HiveType hive, + string id, + Protocol.Registration.RegistrationCatalogEntry catalogEntry, + PackageDetailsCatalogLeaf packageDetails, + NuGetVersion parsedVersion) + { + catalogEntry.Url = packageDetails.Url; + catalogEntry.Type = JsonLdConstants.RegistrationLeafItemCatalogEntryType; + catalogEntry.Authors = packageDetails.Authors ?? string.Empty; + + // Add the "registration" property to each package dependency. + if (packageDetails.DependencyGroups != null) + { + catalogEntry.DependencyGroups = new List(); + foreach (var group in packageDetails.DependencyGroups) + { + var registrationGroup = new RegistrationPackageDependencyGroup + { + Url = group.Url, + Type = group.Type, + TargetFramework = group.TargetFramework, + }; + + catalogEntry.DependencyGroups.Add(registrationGroup); + + if (group.Dependencies == null) + { + continue; + } + + registrationGroup.Dependencies = new List(); + + for (int i = 0; i < group.Dependencies.Count; i++) + { + var catalogDependency = group.Dependencies[i]; + var registrationDependency = new RegistrationPackageDependency + { + Url = catalogDependency.Url, + Type = catalogDependency.Type, + Id = catalogDependency.Id, + Range = catalogDependency.Range, + Registration = _urlBuilder.GetIndexUrl(hive, catalogDependency.Id), + }; + registrationGroup.Dependencies.Add(registrationDependency); + } + } + } + else + { + catalogEntry.DependencyGroups = null; + } + + // Add the types to the deprecation. + if (hive == HiveType.Legacy || hive == HiveType.Gzipped) + { + catalogEntry.Deprecation = null; + } + else + { + catalogEntry.Deprecation = packageDetails.Deprecation; + if (catalogEntry.Deprecation != null) + { + catalogEntry.Deprecation.Type = JsonLdConstants.PackageDeprecationType; + if (catalogEntry.Deprecation.AlternatePackage != null) + { + catalogEntry.Deprecation.AlternatePackage.Type = JsonLdConstants.AlternatePackageType; + } + } + } + + catalogEntry.Description = packageDetails.Description ?? string.Empty; + + if (!string.IsNullOrWhiteSpace(packageDetails.IconUrl) + || !string.IsNullOrWhiteSpace(packageDetails.IconFile)) + { + catalogEntry.IconUrl = GetPackageIconUrl(id, packageDetails); + } + else + { + catalogEntry.IconUrl = string.Empty; + } + + catalogEntry.PackageId = packageDetails.PackageId ?? id; + catalogEntry.Language = packageDetails.Language ?? string.Empty; + catalogEntry.LicenseExpression = packageDetails.LicenseExpression ?? string.Empty; + + if (!string.IsNullOrWhiteSpace(packageDetails.LicenseFile) + || !string.IsNullOrWhiteSpace(packageDetails.LicenseExpression)) + { + // Use the package ID casing from this specific version, since license URLs do not exclusively use + // lowercase package ID like icon URL. This is legacy behavior that can be revisited later. Gallery + // supports case insensitive package IDs so it doesn't matter too much. + catalogEntry.LicenseUrl = LicenseHelper.GetGalleryLicenseUrl( + catalogEntry.PackageId, + parsedVersion.ToNormalizedString(), _galleryBaseUrl); + } + else + { + catalogEntry.LicenseUrl = packageDetails.LicenseUrl ?? string.Empty; + } + + catalogEntry.Listed = packageDetails.IsListed(); + catalogEntry.MinClientVersion = packageDetails.MinClientVersion ?? string.Empty; + catalogEntry.PackageContent = GetPackageContentUrl(id, packageDetails); + catalogEntry.ProjectUrl = packageDetails.ProjectUrl ?? string.Empty; + catalogEntry.Published = packageDetails.Published; + catalogEntry.RequireLicenseAcceptance = packageDetails.RequireLicenseAcceptance ?? false; + catalogEntry.Summary = packageDetails.Summary ?? string.Empty; + + if (packageDetails.Tags != null && packageDetails.Tags.Count > 0) + { + catalogEntry.Tags = packageDetails.Tags; + } + else + { + catalogEntry.Tags = new List { string.Empty }; + } + + catalogEntry.Title = packageDetails.Title ?? string.Empty; + catalogEntry.Version = parsedVersion.ToFullString(); + + if (hive == HiveType.SemVer2 && + packageDetails.Vulnerabilities != null && + packageDetails.Vulnerabilities.Count > 0) + { + catalogEntry.Vulnerabilities = packageDetails.Vulnerabilities.Select(v => + new RegistrationPackageVulnerability() + { + AdvisoryUrl = v.AdvisoryUrl, + Severity = v.Severity + } + ).ToList(); + } + else + { + catalogEntry.Vulnerabilities = null; + } + } + + public RegistrationLeaf NewLeaf(RegistrationLeafItem leafItem) + { + return new RegistrationLeaf + { + Url = leafItem.Url, + Types = JsonLdConstants.RegistrationLeafTypes, + CatalogEntry = leafItem.CatalogEntry.Url, + Listed = leafItem.CatalogEntry.Listed, + PackageContent = leafItem.PackageContent, + Published = leafItem.CatalogEntry.Published, + Registration = leafItem.Registration, + Context = JsonLdConstants.RegistrationLeafContext, + }; + } + + public void UpdateInlinedPageItem(RegistrationPage pageItem, HiveType hive, string id, int count, NuGetVersion lower, NuGetVersion upper) + { + Guard.Assert(pageItem.Items != null, "The provided page item must have inlined leaf items."); + Guard.Assert(pageItem.Items.Count == count, "The provided count must equal the number of leaf items."); + UpdatePage(pageItem, count, lower, upper); + pageItem.Url = _urlBuilder.GetInlinedPageUrl(hive, id, lower, upper); + pageItem.Parent = _urlBuilder.GetIndexUrl(hive, id); + pageItem.Context = null; + } + + public void UpdateNonInlinedPageItem(RegistrationPage pageItem, HiveType hive, string id, int count, NuGetVersion lower, NuGetVersion upper) + { + Guard.Assert(pageItem.Items == null, "The provided page item must not have inlined leaf items."); + UpdatePage(pageItem, count, lower, upper); + pageItem.Url = _urlBuilder.GetPageUrl(hive, id, lower, upper); + pageItem.Parent = null; + pageItem.Context = null; + } + + public void UpdatePage(RegistrationPage page, HiveType hive, string id, int count, NuGetVersion lower, NuGetVersion upper) + { + Guard.Assert(page.Items != null, "Pages must have leaf items."); + Guard.Assert(page.Items.Count == count, "The provided count must equal the number of leaf items."); + UpdatePage(page, count, lower, upper); + page.Url = _urlBuilder.GetPageUrl(hive, id, lower, upper); + page.Parent = _urlBuilder.GetIndexUrl(hive, id); + page.Context = JsonLdConstants.RegistrationContainerContext; + } + + private static void UpdatePage(RegistrationPage page, int count, NuGetVersion lower, NuGetVersion upper) + { + Guard.Assert(count > 0, "Page count must be greater than zero."); + Guard.Assert(lower <= upper, "The lower bound on a page must be less than or equal to the upper bound."); + page.Type = JsonLdConstants.RegistrationPageType; + page.Lower = lower.ToNormalizedString(); + page.Upper = upper.ToNormalizedString(); + page.Count = count; + } + + public void UpdateIndex(RegistrationIndex index, HiveType hive, string id, int count) + { + Guard.Assert(count > 0, "Indexes must have at least one page item."); + Guard.Assert(index.Items.Count == count, "The provided count must equal the number of page items."); + index.Url = _urlBuilder.GetIndexUrl(hive, id); + index.Types = JsonLdConstants.RegistrationIndexTypes; + index.Count = count; + index.Context = JsonLdConstants.RegistrationContainerContext; + } + + public void UpdateIndexUrls(RegistrationIndex index, HiveType fromHive, HiveType toHive) + { + index.Url = _urlBuilder.ConvertHive(fromHive, toHive, index.Url); + + foreach (var pageItem in index.Items) + { + UpdatePageUrls(pageItem, fromHive, toHive); + } + } + + public void UpdatePageUrls(RegistrationPage page, HiveType fromHive, HiveType toHive) + { + page.Url = _urlBuilder.ConvertHive(fromHive, toHive, page.Url); + + if (page.Parent != null) + { + page.Parent = _urlBuilder.ConvertHive(fromHive, toHive, page.Parent); + } + + if (page.Items != null) + { + foreach (var item in page.Items) + { + UpdateLeafItemUrls(item, fromHive, toHive); + } + } + } + + private void UpdateLeafItemUrls(RegistrationLeafItem leafItem, HiveType fromHive, HiveType toHive) + { + leafItem.Url = _urlBuilder.ConvertHive(fromHive, toHive, leafItem.Url); + leafItem.Registration = _urlBuilder.ConvertHive(fromHive, toHive, leafItem.Registration); + + if (leafItem.CatalogEntry.DependencyGroups != null) + { + foreach (var dependencyGroup in leafItem.CatalogEntry.DependencyGroups) + { + if (dependencyGroup.Dependencies == null) + { + continue; + } + + foreach (var dependency in dependencyGroup.Dependencies) + { + dependency.Registration = _urlBuilder.ConvertHive(fromHive, toHive, dependency.Registration); + } + } + } + } + + public void UpdateLeafUrls(RegistrationLeaf leaf, HiveType fromHive, HiveType toHive) + { + leaf.Url = _urlBuilder.ConvertHive(fromHive, toHive, leaf.Url); + leaf.Registration = _urlBuilder.ConvertHive(fromHive, toHive, leaf.Registration); + } + + public void UpdateCommit(ICommitted committed, CatalogCommit commit) + { + committed.CommitId = commit.Id; + committed.CommitTimestamp = commit.Timestamp; + } + + private string GetPackageIconUrl(string id, PackageDetailsCatalogLeaf packageDetails) + { + return new Uri(_flatContainerPathProvider.GetIconPath(id, packageDetails.PackageVersion)).AbsoluteUri; + } + + private string GetPackageContentUrl(string id, PackageDetailsCatalogLeaf packageDetails) + { + return new Uri(_flatContainerPathProvider.GetPackagePath(id, packageDetails.PackageVersion)).AbsoluteUri; + } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Schema/IEntityBuilder.cs b/src/NuGet.Jobs.Catalog2Registration/Schema/IEntityBuilder.cs new file mode 100644 index 000000000..21914a0a0 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Schema/IEntityBuilder.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Protocol.Catalog; +using NuGet.Protocol.Registration; +using NuGet.Versioning; + +namespace NuGet.Jobs.Catalog2Registration +{ + public interface IEntityBuilder + { + void UpdateLeafItem(RegistrationLeafItem existing, HiveType hive, string id, PackageDetailsCatalogLeaf packageDetails); + RegistrationLeaf NewLeaf(RegistrationLeafItem leafItem); + void UpdateCommit(ICommitted committed, CatalogCommit commit); + void UpdateInlinedPageItem(RegistrationPage pageItem, HiveType hive, string id, int count, NuGetVersion lower, NuGetVersion upper); + void UpdateNonInlinedPageItem(RegistrationPage pageItem, HiveType hive, string id, int count, NuGetVersion lower, NuGetVersion upper); + void UpdatePage(RegistrationPage pageItem, HiveType hive, string id, int count, NuGetVersion lower, NuGetVersion upper); + void UpdateIndex(RegistrationIndex index, HiveType hive, string id, int count); + void UpdateIndexUrls(RegistrationIndex index, HiveType fromHive, HiveType toHive); + void UpdatePageUrls(RegistrationPage page, HiveType fromHive, HiveType toHive); + void UpdateLeafUrls(RegistrationLeaf leaf, HiveType fromHive, HiveType toHive); + } +} \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2Registration/Schema/JsonLdConstants.cs b/src/NuGet.Jobs.Catalog2Registration/Schema/JsonLdConstants.cs new file mode 100644 index 000000000..4166f9f1a --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Schema/JsonLdConstants.cs @@ -0,0 +1,123 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using NuGet.Protocol.Catalog; +using NuGet.Protocol.Registration; + +namespace NuGet.Jobs.Catalog2Registration +{ + public static class JsonLdConstants + { + public static readonly List RegistrationIndexTypes = new List + { + "catalog:CatalogRoot", + "PackageRegistration", + "catalog:Permalink" + }; + + public static readonly string RegistrationPageType = "catalog:CatalogPage"; + + public static readonly string RegistrationLeafItemType = "Package"; + + public static readonly List RegistrationLeafTypes = new List + { + "Package", + "http://schema.nuget.org/catalog#Permalink" + }; + + public static readonly string RegistrationLeafItemCatalogEntryType = "PackageDetails"; + + public static readonly string PackageDeprecationType = "deprecation"; + + public static readonly string AlternatePackageType = "alternatePackage"; + + public static readonly RegistrationContainerContext RegistrationContainerContext = new RegistrationContainerContext + { + Vocab = "http://schema.nuget.org/schema#", + Catalog = "http://schema.nuget.org/catalog#", + Xsd = "http://www.w3.org/2001/XMLSchema#", + Items = new ContextTypeDescription + { + Id = "catalog:item", + Container = "@set", + }, + CommitTimestamp = new ContextTypeDescription + { + Id = "catalog:commitTimeStamp", + Type = "xsd:dateTime", + }, + CommitId = new ContextTypeDescription + { + Id = "catalog:commitId", + }, + Count = new ContextTypeDescription + { + Id = "catalog:count", + }, + Parent = new ContextTypeDescription + { + Id = "catalog:parent", + Type = "@id", + }, + Tags = new ContextTypeDescription + { + Container = "@set", + Id = "tag", + }, + Reasons = new ContextTypeDescription + { + Container = "@set" + }, + PackageTargetFrameworks = new ContextTypeDescription + { + Container = "@set", + Id = "packageTargetFramework", + }, + DependencyGroups = new ContextTypeDescription + { + Container = "@set", + Id = "dependencyGroup", + }, + Dependencies = new ContextTypeDescription + { + Container = "@set", + Id = "dependency", + }, + PackageContent = new ContextTypeDescription + { + Type = "@id", + }, + Published = new ContextTypeDescription + { + Type = "xsd:dateTime", + }, + Registration = new ContextTypeDescription + { + Type = "@id", + }, + }; + + public static readonly RegistrationLeafContext RegistrationLeafContext = new RegistrationLeafContext + { + Vocab = "http://schema.nuget.org/schema#", + Xsd = "http://www.w3.org/2001/XMLSchema#", + CatalogEntry = new ContextTypeDescription + { + Type = "@id", + }, + Registration = new ContextTypeDescription + { + Type = "@id", + }, + PackageContent = new ContextTypeDescription + { + Type = "@id", + }, + Published = new ContextTypeDescription + { + Type = "xsd:dateTime", + }, + }; + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Schema/RegistrationUrlBuilder.cs b/src/NuGet.Jobs.Catalog2Registration/Schema/RegistrationUrlBuilder.cs new file mode 100644 index 000000000..ab2113b35 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Schema/RegistrationUrlBuilder.cs @@ -0,0 +1,145 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Extensions.Options; +using NuGet.Services; +using NuGet.Versioning; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class RegistrationUrlBuilder + { + private readonly IOptionsSnapshot _options; + private readonly string _legacyBaseUrl; + private readonly string _gzippedBaseUrl; + private readonly string _semver2BaseUrl; + + public RegistrationUrlBuilder(IOptionsSnapshot options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + + _legacyBaseUrl = EnsureTrailingSlash(_options.Value.LegacyBaseUrl); + _gzippedBaseUrl = EnsureTrailingSlash(_options.Value.GzippedBaseUrl); + _semver2BaseUrl = EnsureTrailingSlash(_options.Value.SemVer2BaseUrl); + } + + public string GetIndexPath(string id) + { + if (id == null) + { + throw new ArgumentNullException(nameof(id)); + } + + return $"{Uri.EscapeUriString(id.ToLowerInvariant())}/index.json"; + } + + public string GetIndexUrl(HiveType hive, string id) + { + return GetBaseUrl(hive) + GetIndexPath(id); + } + + public string GetInlinedPageUrl(HiveType hive, string id, NuGetVersion lower, NuGetVersion upper) + { + return GetIndexUrl(hive, id) + "#" + GetPageFragment(lower, upper); + } + + public string ConvertHive(HiveType fromHive, HiveType toHive, string url) + { + var path = ConvertToPath(fromHive, url); + return GetBaseUrl(toHive) + path; + } + + public string ConvertToPath(HiveType hive, string url) + { + if (url == null) + { + throw new ArgumentNullException(nameof(url)); + } + + var baseUrl = GetBaseUrl(hive); + Guard.Assert(url.StartsWith(baseUrl), $"URL '{url}' does not start with expected base URL '{baseUrl}'."); + return url.Substring(baseUrl.Length); + } + + public string GetPageUrl(HiveType hive, string id, NuGetVersion lower, NuGetVersion upper) + { + return GetBaseUrl(hive) + GetPagePath(id, lower, upper); + } + + public string GetPagePath(string id, NuGetVersion lower, NuGetVersion upper) + { + if (id == null) + { + throw new ArgumentNullException(nameof(id)); + } + + return $"{Uri.EscapeUriString(id.ToLowerInvariant())}/{GetPageFragment(lower, upper)}.json"; + } + + private static string GetPageFragment(NuGetVersion lower, NuGetVersion upper) + { + if (lower == null) + { + throw new ArgumentNullException(nameof(lower)); + } + + if (upper == null) + { + throw new ArgumentNullException(nameof(upper)); + } + + return + $"page/" + + $"{Uri.EscapeUriString(lower.ToNormalizedString().ToLowerInvariant())}/" + + $"{Uri.EscapeUriString(upper.ToNormalizedString().ToLowerInvariant())}"; + } + + public string GetLeafPath(string id, NuGetVersion version) + { + if (id == null) + { + throw new ArgumentNullException(nameof(id)); + } + + if (version == null) + { + throw new ArgumentNullException(nameof(version)); + } + + return + $"{Uri.EscapeUriString(id.ToLowerInvariant())}/" + + $"{Uri.EscapeUriString(version.ToNormalizedString().ToLowerInvariant())}.json"; + } + + public string GetLeafUrl(HiveType hive, string id, NuGetVersion version) + { + return GetBaseUrl(hive) + GetLeafPath(id, version); + } + + private string GetBaseUrl(HiveType hive) + { + switch (hive) + { + case HiveType.Legacy: + return _legacyBaseUrl; + case HiveType.Gzipped: + return _gzippedBaseUrl; + case HiveType.SemVer2: + return _semver2BaseUrl; + default: + throw new NotImplementedException($"The hive type '{hive}' does not have a configured base URL."); + } + } + + private static string EnsureTrailingSlash(string baseUrl) + { + if (baseUrl == null) + { + throw new ArgumentNullException(nameof(baseUrl)); + } + + return baseUrl.TrimEnd('/') + '/'; + } + } +} diff --git a/src/NuGet.Jobs.Catalog2Registration/Scripts/Functions.ps1 b/src/NuGet.Jobs.Catalog2Registration/Scripts/Functions.ps1 new file mode 100644 index 000000000..a8bff40fc --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Scripts/Functions.ps1 @@ -0,0 +1,30 @@ +Function Uninstall-NuGetService() { + Param ([string]$ServiceName) + + if (Get-Service $ServiceName -ErrorAction SilentlyContinue) + { + Write-Host Removing service $ServiceName... + Stop-Service $ServiceName -Force + sc.exe delete $ServiceName + Write-Host Removed service $ServiceName. + } else { + Write-Host Skipping removal of service $ServiceName - no such service exists. + } +} + +Function Install-NuGetService() { + Param ([string]$ServiceName, [string]$ServiceTitle, [string]$ScriptToRun) + + Write-Host Installing service $ServiceName... + + $installService = "nssm install $ServiceName $ScriptToRun" + cmd /C $installService + + Set-Service -Name $ServiceName -DisplayName "$ServiceTitle - $ServiceName" -Description "Runs $ServiceTitle." -StartupType Automatic + sc.exe failure $ServiceName reset= 30 actions= restart/5000 + + # Run service + net start $ServiceName + + Write-Host Installed service $ServiceName. +} \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2Registration/Scripts/PostDeploy.ps1 b/src/NuGet.Jobs.Catalog2Registration/Scripts/PostDeploy.ps1 new file mode 100644 index 000000000..7d5183d5b --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Scripts/PostDeploy.ps1 @@ -0,0 +1,18 @@ +. .\Functions.ps1 + +$jobsToInstall = $OctopusParameters["Jobs.ServiceNames"].Split("{,}") + +Write-Host Installing services... + +$currentDirectory = [string](Get-Location) + +$jobsToInstall.Split("{;}") | %{ + $serviceName = $_ + $serviceTitle = $OctopusParameters["Jobs.$serviceName.Title"] + $scriptToRun = $OctopusParameters["Jobs.$serviceName.Script"] + $scriptToRun = "$currentDirectory\$scriptToRun" + + Install-NuGetService -ServiceName $serviceName -ServiceTitle $serviceTitle -ScriptToRun $scriptToRun +} + +Write-Host Installed services. \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2Registration/Scripts/PreDeploy.ps1 b/src/NuGet.Jobs.Catalog2Registration/Scripts/PreDeploy.ps1 new file mode 100644 index 000000000..ef711a912 --- /dev/null +++ b/src/NuGet.Jobs.Catalog2Registration/Scripts/PreDeploy.ps1 @@ -0,0 +1,11 @@ +. .\Functions.ps1 + +$jobsToInstall = $OctopusParameters["Jobs.ServiceNames"].Split("{,}") + +Write-Host Removing services... + +$jobsToInstall.Split("{;}") | %{ + Uninstall-NuGetService -ServiceName $_ +} + +Write-Host Removed services. \ No newline at end of file diff --git a/src/NuGet.Jobs.Catalog2Registration/Scripts/nssm.exe b/src/NuGet.Jobs.Catalog2Registration/Scripts/nssm.exe new file mode 100644 index 000000000..6ccfe3cfb Binary files /dev/null and b/src/NuGet.Jobs.Catalog2Registration/Scripts/nssm.exe differ diff --git a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj index 5191f89ea..82f93efc9 100644 --- a/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj +++ b/src/NuGet.Jobs.Common/NuGet.Jobs.Common.csproj @@ -104,16 +104,16 @@ all - 2.74.0 + 2.75.0 - 2.74.0 + 2.75.0 - 2.74.0 + 2.75.0 - 2.74.0 + 2.75.0 diff --git a/src/NuGet.Jobs.Db2AzureSearch/App.config b/src/NuGet.Jobs.Db2AzureSearch/App.config new file mode 100644 index 000000000..56efbc7b5 --- /dev/null +++ b/src/NuGet.Jobs.Db2AzureSearch/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/NuGet.Jobs.Db2AzureSearch/Job.cs b/src/NuGet.Jobs.Db2AzureSearch/Job.cs new file mode 100644 index 000000000..7ee5090af --- /dev/null +++ b/src/NuGet.Jobs.Db2AzureSearch/Job.cs @@ -0,0 +1,31 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using NuGet.Services.AzureSearch; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.AzureSearch.Db2AzureSearch; + +namespace NuGet.Jobs +{ + public class Job : AzureSearchJob + { + private const string ConfigurationSectionName = "Db2AzureSearch"; + private const string DevelopmentConfigurationSectionName = "Db2AzureSearch:Development"; + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + base.ConfigureJobServices(services, configurationRoot); + + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + services.Configure(configurationRoot.GetSection(ConfigurationSectionName)); + services.Configure( + configurationRoot.GetSection(DevelopmentConfigurationSectionName)); + services.Configure( + configurationRoot.GetSection(DevelopmentConfigurationSectionName)); + } + } +} diff --git a/src/NuGet.Jobs.Db2AzureSearch/NuGet.Jobs.Db2AzureSearch.csproj b/src/NuGet.Jobs.Db2AzureSearch/NuGet.Jobs.Db2AzureSearch.csproj new file mode 100644 index 000000000..a22cc2db6 --- /dev/null +++ b/src/NuGet.Jobs.Db2AzureSearch/NuGet.Jobs.Db2AzureSearch.csproj @@ -0,0 +1,78 @@ + + + + + + Debug + AnyCPU + {209B1B7F-1C5C-41EC-B6A6-E01FD9C86E26} + Exe + NuGet.Jobs + NuGet.Jobs.Db2AzureSearch + v4.7.2 + 512 + true + true + PackageReference + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + 0.3.0 + runtime; build; native; contentfiles; analyzers + all + + + + + {4B4B1EFB-8F33-42E6-B79F-54E7F3293D31} + NuGet.Jobs.Common + + + {1a53fe3d-8041-4773-942f-d73aef5b82b2} + NuGet.Services.AzureSearch + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + + \ No newline at end of file diff --git a/src/NuGet.Jobs.Db2AzureSearch/NuGet.Jobs.Db2AzureSearch.nuspec b/src/NuGet.Jobs.Db2AzureSearch/NuGet.Jobs.Db2AzureSearch.nuspec new file mode 100644 index 000000000..ced5bdd1d --- /dev/null +++ b/src/NuGet.Jobs.Db2AzureSearch/NuGet.Jobs.Db2AzureSearch.nuspec @@ -0,0 +1,15 @@ + + + + Db2AzureSearch + $version$ + .NET Foundation + .NET Foundation + Db2AzureSearch + Copyright .NET Foundation + + + + + + \ No newline at end of file diff --git a/src/NuGet.Jobs.Db2AzureSearch/Program.cs b/src/NuGet.Jobs.Db2AzureSearch/Program.cs new file mode 100644 index 000000000..6fcb99491 --- /dev/null +++ b/src/NuGet.Jobs.Db2AzureSearch/Program.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Jobs +{ + public class Program + { + public static int Main(string[] args) + { + var job = new Job(); + return JobRunner.RunOnce(job, args).GetAwaiter().GetResult(); + } + } +} diff --git a/src/NuGet.Jobs.Db2AzureSearch/Properties/AssemblyInfo.cs b/src/NuGet.Jobs.Db2AzureSearch/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..e0d6c9160 --- /dev/null +++ b/src/NuGet.Jobs.Db2AzureSearch/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Jobs.Db2AzureSearch")] +[assembly: ComVisible(false)] +[assembly: Guid("209b1b7f-1c5c-41ec-b6a6-e01fd9c86e26")] diff --git a/src/NuGet.Jobs.Db2AzureSearch/README.md b/src/NuGet.Jobs.Db2AzureSearch/README.md new file mode 100644 index 000000000..1712ac794 --- /dev/null +++ b/src/NuGet.Jobs.Db2AzureSearch/README.md @@ -0,0 +1,124 @@ +## Overview + +**Subsystem: Search 🔎** + +This tool creates the resources needed to run the [NuGet search service](../NuGet.Services.SearchService). These resources can be updated using the [Catalog2AzureSearch](../NuGet.Jobs.Catalog2AzureSearch) and [Auxiliary2AzureSearch](../NuGet.Jobs.Auxiliary2AzureSearch) jobs. + +Specifically, this tool creates: + +* The [Azure Search indexes](../../docs/Azure-Search-indexes.md) +* The [search auxiliary files](../../docs/Search-auxiliary-files.md) +* The [search version list resource](../../docs/Search-version-list-resource.md) + +## Running the job + +You can run this job using: + +```ps1 +NuGet.Jobs.Db2AzureSearch.exe -Configuration path\to\your\settings.json +``` + +### Using DEV resources + +The easiest way to run the tool if you are on the nuget.org team is to use the DEV environment resources: + +1. Install the certificate used to authenticate as our client AAD app registration into your `CurrentUser` certificate store. +1. Clone our internal [`NuGetDeployment`](https://nuget.visualstudio.com/DefaultCollection/NuGetMicrosoft/_git/NuGetDeploymentp) repository. +1. Update your cloned copy of the [DEV Db2AzureSearch appsettings.json](https://nuget.visualstudio.com/DefaultCollection/NuGetMicrosoft/_git/NuGetDeployment?path=%2Fsrc%2FJobs%2FNuGet.Jobs.Cloud%2FJobs%2FDb2AzureSearch%2FDEV%2Fnorthcentralus%2Fappsettings.json) file to authenticate using the certificate you installed: +```json +{ + ... + "KeyVault_VaultName": "PLACEHOLDER", + "KeyVault_ClientId": "PLACEHOLDER", + "KeyVault_CertificateThumbprint": "PLACEHOLDER", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "CurrentUser" + ... +} +``` + +1. Update the `-Configuration` CLI option to point to the DEV Azure Search settings: `NuGetDeployment/src/Jobs/NuGet.Jobs.Cloud/Jobs/Db2AzureSearch/DEV/northcentralus/appsettings.json` + +### Using personal Azure resources + +As an alternative to using nuget.org's DEV resources, you can also run this tool using your personal Azure resources. + +#### Prerequisites + +- **Gallery DB**. This can be initialized locally using the [NuGetGallery](https://github.com/NuGet/NuGetGallery/blob/master/README.md). +- **Azure Search**. You can create your own Azure Search resource using the [Azure Portal](https://docs.microsoft.com/en-us/azure/search/search-create-service-portal). +- **Azure Blob Storage**. You can create your own Azure Blob Storage using the [Azure Portal](https://docs.microsoft.com/en-us/azure/storage/common/storage-account-create). + +In your Azure Blob Storage account, you will need to create a container named `ng-search-data` and upload the following files: +1. `downloads.v1.json` with content `[]` +1. `ExcludedPackages.v1.json` with content `[]` +1. `verifiedPackages.json` with content `[]` + +If you are on the nuget.org team, you can copy these files from the [PROD auxiliary files container](https://nuget.visualstudio.com/DefaultCollection/NuGetMicrosoft/_git/NuGetDeployment?path=%2Fsrc%2FJobs%2FNuGet.Jobs.Cloud%2FJobs%2FDb2AzureSearch%2FPROD%2Fnorthcentralus%2Fappsettings.json&version=GBmaster&line=18&lineEnd=24&lineStartColumn=1&lineEndColumn=1&lineStyle=plain). + +#### Settings + +Once you've created your Azure resources, you can create your `settings.json` file. There's a few `PLACEHOLDER` values you will need to fill in yourself: + +* The `GalleryDb:ConnectionString` setting is the connection string to your Gallery DB. +* The `SearchServiceName` setting is the name of your Azure Search resource. For example, use the name `foo-bar` for the Azure Search service with URL `https://foo-bar.search.windows.net`. +* The `SearchServiceApiKey` setting is an admin key that has write permissions to the Azure Search resource. +* The `StorageConnectionString` and `AuxiliaryDataStorageConnectionString` settings are both the connection string to your Azure Blob Storage account. + +```json +{ + "GalleryDb": { + "ConnectionString": "PLACEHOLDER" + }, + + "Db2AzureSearch": { + "AzureSearchBatchSize": 1000, + "MaxConcurrentBatches": 4, + "MaxConcurrentVersionListWriters": 8, + "SearchServiceName": "PLACEHOLDER", + "SearchServiceApiKey": "PLACEHOLDER", + "SearchIndexName": "search-000", + "HijackIndexName": "hijack-000", + "StorageConnectionString": "PLACEHOLDER", + "StorageContainer": "v3-azuresearch-000", + "StoragePath": "", + "GalleryBaseUrl": "https://www.nuget.org/", + "AuxiliaryDataStorageConnectionString": "PLACEHOLDER", + "AuxiliaryDataStorageContainer": "ng-search-data", + "AuxiliaryDataStorageDownloadsPath": "downloads.v1.json", + "AuxiliaryDataStorageExcludedPackagesPath": "ExcludedPackages.v1.json", + "AuxiliaryDataStorageVerifiedPackagesPath": "verifiedPackages.json", + "FlatContainerBaseUrl": "https://api.nuget.org/", + "FlatContainerContainerName": "v3-flatcontainer", + "AllIconsInFlatContainer": false, + "DatabaseBatchSize": 10000, + "CatalogIndexUrl": "https://api.nuget.org/v3/catalog0/index.json", + "EnablePopularityTransfers": true, + "Scoring": { + "FieldWeights": { + "PackageId": 9, + "TokenizedPackageId": 9, + "Tags": 5 + }, + "DownloadScoreBoost": 30000, + "PopularityTransfer": 0.99 + } + } +} +``` + +## Algorithm + +At a high-level, here's how Db2AzureSearch works: + +1. Create the [Azure Search indexes](../../docs/Azure-Search-indexes.md) +1. Create the Azure Blob storage container for the [search auxiliary files](../../docs/Search-auxiliary-files.md) +1. Capture the catalog's cursor +1. Load initial data from Gallery DB and statistics auxiliary files +1. Process package metadata in batches + 1. Load a chunk of packages from Gallery DB + 1. Generate and upload documents to the Azure Search indexes + 1. Update the [search version list resource](../../docs/Search-version-list-resource.md) +1. Write the [search auxiliary files](../../docs/Search-auxiliary-files.md) to search storage +1. Write the catalog's cursor to search storage \ No newline at end of file diff --git a/src/NuGet.Jobs.Db2AzureSearch/Scripts/PostDeploy.ps1 b/src/NuGet.Jobs.Db2AzureSearch/Scripts/PostDeploy.ps1 new file mode 100644 index 000000000..bfa990c26 --- /dev/null +++ b/src/NuGet.Jobs.Db2AzureSearch/Scripts/PostDeploy.ps1 @@ -0,0 +1,5 @@ +.\RunJob.cmd + +if ($LastExitCode -ne 0) { + throw "The job failed with exit code $LastExitCode" +} diff --git a/src/NuGet.Protocol.Catalog/CatalogClient.cs b/src/NuGet.Protocol.Catalog/CatalogClient.cs new file mode 100644 index 000000000..1a5fa81a4 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/CatalogClient.cs @@ -0,0 +1,90 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Protocol.Catalog +{ + public class CatalogClient : ICatalogClient + { + private readonly ISimpleHttpClient _jsonClient; + private readonly ILogger _logger; + + public CatalogClient(ISimpleHttpClient jsonClient, ILogger logger) + { + _jsonClient = jsonClient ?? throw new ArgumentNullException(nameof(jsonClient)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task GetIndexAsync(string indexUrl) + { + var result = await _jsonClient.DeserializeUrlAsync(indexUrl); + return result.GetResultOrThrow(); + } + + public async Task GetPageAsync(string pageUrl) + { + var result = await _jsonClient.DeserializeUrlAsync(pageUrl); + return result.GetResultOrThrow(); + } + + public async Task GetLeafAsync(string leafUrl) + { + // Buffer all of the JSON so we can parse twice. Once to determine the leaf type and once to deserialize + // the entire thing to the proper leaf type. + _logger.LogDebug("Downloading {leafUrl} as a byte array.", leafUrl); + var jsonBytes = await _jsonClient.GetByteArrayAsync(leafUrl); + var untypedLeaf = _jsonClient.DeserializeBytes(jsonBytes); + + switch (untypedLeaf.Type) + { + case CatalogLeafType.PackageDetails: + return _jsonClient.DeserializeBytes(jsonBytes); + case CatalogLeafType.PackageDelete: + return _jsonClient.DeserializeBytes(jsonBytes); + default: + throw new NotSupportedException($"The catalog leaf type '{untypedLeaf.Type}' is not supported."); + } + } + + private async Task GetLeafAsync(CatalogLeafType type, string leafUrl) + { + switch (type) + { + case CatalogLeafType.PackageDetails: + return await GetPackageDetailsLeafAsync(leafUrl); + case CatalogLeafType.PackageDelete: + return await GetPackageDeleteLeafAsync(leafUrl); + default: + throw new NotSupportedException($"The catalog leaf type '{type}' is not supported."); + } + } + + public Task GetPackageDeleteLeafAsync(string leafUrl) + { + return GetAndValidateLeafAsync(CatalogLeafType.PackageDelete, leafUrl); + } + + public Task GetPackageDetailsLeafAsync(string leafUrl) + { + return GetAndValidateLeafAsync(CatalogLeafType.PackageDetails, leafUrl); + } + + private async Task GetAndValidateLeafAsync(CatalogLeafType type, string leafUrl) where T : CatalogLeaf + { + var result = await _jsonClient.DeserializeUrlAsync(leafUrl); + var leaf = result.GetResultOrThrow(); + + if (leaf.Type != type) + { + throw new ArgumentException( + $"The leaf type found in the document does not match the expected '{type}' type.", + nameof(type)); + } + + return leaf; + } + } +} diff --git a/src/NuGet.Protocol.Catalog/CatalogProcessor.cs b/src/NuGet.Protocol.Catalog/CatalogProcessor.cs new file mode 100644 index 000000000..e798f969a --- /dev/null +++ b/src/NuGet.Protocol.Catalog/CatalogProcessor.cs @@ -0,0 +1,218 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Protocol.Core.Types; + +namespace NuGet.Protocol.Catalog +{ + public class CatalogProcessor + { + private const string CatalogResourceType = "Catalog/3.0.0"; + private readonly ICatalogLeafProcessor _leafProcessor; + private readonly ICatalogClient _client; + private readonly ICursor _cursor; + private readonly ILogger _logger; + private readonly CatalogProcessorSettings _settings; + + public CatalogProcessor( + ICursor cursor, + ICatalogClient client, + ICatalogLeafProcessor leafProcessor, + CatalogProcessorSettings settings, + ILogger logger) + { + _leafProcessor = leafProcessor ?? throw new ArgumentNullException(nameof(leafProcessor)); + _client = client ?? throw new ArgumentNullException(nameof(client)); + _cursor = cursor ?? throw new ArgumentNullException(nameof(cursor)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (settings == null) + { + throw new ArgumentNullException(nameof(settings)); + } + + if (settings.ServiceIndexUrl == null) + { + throw new ArgumentException( + $"The {nameof(CatalogProcessorSettings.ServiceIndexUrl)} property of the " + + $"{nameof(CatalogProcessorSettings)} must not be null.", + nameof(settings)); + } + + // Clone the settings to avoid mutability issues. + _settings = settings.Clone(); + } + + /// + /// Discovers and downloads all of the catalog leafs after the current cursor value and before the maximum + /// commit timestamp found in the settings. Each catalog leaf is passed to the catalog leaf processor in + /// chronological order. After a commit is completed, its commit timestamp is written to the cursor, i.e. when + /// transitioning from commit timestamp A to B, A is written to the cursor so that it never is processed again. + /// + /// True if all of the catalog leaves found were processed successfully. + public async Task ProcessAsync() + { + var catalogIndexUrl = await GetCatalogIndexUrlAsync(); + + var minCommitTimestamp = await GetMinCommitTimestamp(); + _logger.LogInformation( + "Using time bounds {min:O} (exclusive) to {max:O} (inclusive).", + minCommitTimestamp, + _settings.MaxCommitTimestamp); + + return await ProcessIndexAsync(catalogIndexUrl, minCommitTimestamp); + } + + private async Task ProcessIndexAsync(string catalogIndexUrl, DateTimeOffset minCommitTimestamp) + { + var index = await _client.GetIndexAsync(catalogIndexUrl); + + var pageItems = index.GetPagesInBounds( + minCommitTimestamp, + _settings.MaxCommitTimestamp); + _logger.LogInformation( + "{pages} pages were in the time bounds, out of {totalPages}.", + pageItems.Count, + index.Items.Count); + + var success = true; + for (var i = 0; i < pageItems.Count; i++) + { + success = await ProcessPageAsync(minCommitTimestamp, pageItems[i]); + if (!success) + { + _logger.LogWarning( + "{unprocessedPages} out of {pages} pages were left incomplete due to a processing failure.", + pageItems.Count - i, + pageItems.Count); + break; + } + } + + return success; + } + + private async Task ProcessPageAsync(DateTimeOffset minCommitTimestamp, CatalogPageItem pageItem) + { + var page = await _client.GetPageAsync(pageItem.Url); + + var leafItems = page.GetLeavesInBounds( + minCommitTimestamp, + _settings.MaxCommitTimestamp, + _settings.ExcludeRedundantLeaves); + _logger.LogInformation( + "On page {page}, {leaves} out of {totalLeaves} were in the time bounds.", + pageItem.Url, + leafItems.Count, + page.Items.Count); + + DateTimeOffset? newCursor = null; + var success = true; + for (var i = 0; i < leafItems.Count; i++) + { + var leafItem = leafItems[i]; + + if (newCursor.HasValue && newCursor.Value != leafItem.CommitTimestamp) + { + await _cursor.SetAsync(newCursor.Value); + } + + newCursor = leafItem.CommitTimestamp; + + success = await ProcessLeafAsync(leafItem); + if (!success) + { + _logger.LogWarning( + "{unprocessedLeaves} out of {leaves} leaves were left incomplete due to a processing failure.", + leafItems.Count - i, + leafItems.Count); + break; + } + } + + if (newCursor.HasValue && success) + { + await _cursor.SetAsync(newCursor.Value); + } + + return success; + } + + private async Task ProcessLeafAsync(CatalogLeafItem leafItem) + { + bool success; + try + { + switch (leafItem.Type) + { + case CatalogLeafType.PackageDelete: + var packageDelete = await _client.GetPackageDeleteLeafAsync(leafItem.Url); + success = await _leafProcessor.ProcessPackageDeleteAsync(packageDelete); + break; + case CatalogLeafType.PackageDetails: + var packageDetails = await _client.GetPackageDetailsLeafAsync(leafItem.Url); + success = await _leafProcessor.ProcessPackageDetailsAsync(packageDetails); + break; + default: + throw new NotSupportedException($"The catalog leaf type '{leafItem.Type}' is not supported."); + } + } + catch (Exception exception) + { + _logger.LogError( + 0, + exception, + "An exception was thrown while processing leaf {leafUrl}.", + leafItem.Url); + success = false; + } + + if (!success) + { + _logger.LogWarning( + "Failed to process leaf {leafUrl} ({packageId} {packageVersion}, {leafType}).", + leafItem.Url, + leafItem.PackageId, + leafItem.PackageVersion, + leafItem.Type); + } + + return success; + } + + private async Task GetMinCommitTimestamp() + { + var minCommitTimestamp = await _cursor.GetAsync(); + + minCommitTimestamp = minCommitTimestamp + ?? _settings.DefaultMinCommitTimestamp + ?? _settings.MinCommitTimestamp; + + if (minCommitTimestamp.Value < _settings.MinCommitTimestamp) + { + minCommitTimestamp = _settings.MinCommitTimestamp; + } + + return minCommitTimestamp.Value; + } + + private async Task GetCatalogIndexUrlAsync() + { + _logger.LogInformation("Getting catalog index URL from {serviceIndexUrl}.", _settings.ServiceIndexUrl); + string catalogIndexUrl; + var sourceRepository = Repository.Factory.GetCoreV3(_settings.ServiceIndexUrl, FeedType.HttpV3); + var serviceIndexResource = await sourceRepository.GetResourceAsync(); + catalogIndexUrl = serviceIndexResource.GetServiceEntryUri(CatalogResourceType)?.AbsoluteUri; + if (catalogIndexUrl == null) + { + throw new InvalidOperationException( + $"The service index does not contain resource '{CatalogResourceType}'."); + } + + return catalogIndexUrl; + } + } +} diff --git a/src/NuGet.Protocol.Catalog/CatalogProcessorSettings.cs b/src/NuGet.Protocol.Catalog/CatalogProcessorSettings.cs new file mode 100644 index 000000000..7c61a1836 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/CatalogProcessorSettings.cs @@ -0,0 +1,61 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Protocol.Catalog +{ + /// + /// Settings for how should behave. Defaults to processing all catalog items on + /// . + /// + public class CatalogProcessorSettings + { + public CatalogProcessorSettings() + { + ServiceIndexUrl = "https://api.nuget.org/v3/index.json"; + DefaultMinCommitTimestamp = null; + MinCommitTimestamp = DateTimeOffset.MinValue; + MaxCommitTimestamp = DateTimeOffset.MaxValue; + ExcludeRedundantLeaves = true; + } + + internal CatalogProcessorSettings Clone() + { + return new CatalogProcessorSettings + { + ServiceIndexUrl = ServiceIndexUrl, + DefaultMinCommitTimestamp = DefaultMinCommitTimestamp, + MinCommitTimestamp = MinCommitTimestamp, + MaxCommitTimestamp = MaxCommitTimestamp, + ExcludeRedundantLeaves = ExcludeRedundantLeaves, + }; + } + + /// + /// The service index to discover the catalog index URL. + /// + public string ServiceIndexUrl { get; set; } + + /// + /// The minimum commit timestamp to use when no cursor value has been saved. + /// + public DateTimeOffset? DefaultMinCommitTimestamp { get; set; } + + /// + /// The absolute minimum (exclusive) commit timestamp to process in the catalog. + /// + public DateTimeOffset MinCommitTimestamp { get; set; } + + /// + /// The absolute maximum (inclusive) commit timestamp to process in the catalog. + /// + public DateTimeOffset MaxCommitTimestamp { get; set; } + + /// + /// If multiple catalog leaves are found in a page concerning the same package ID and version, only the latest + /// is processed. + /// + public bool ExcludeRedundantLeaves { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/FileCursor.cs b/src/NuGet.Protocol.Catalog/FileCursor.cs new file mode 100644 index 000000000..d36c0bf7c --- /dev/null +++ b/src/NuGet.Protocol.Catalog/FileCursor.cs @@ -0,0 +1,58 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + /// + /// A cursor implementation which stores the cursor in local file. The cursor value is written to the file as + /// a JSON object. + /// + public class FileCursor : ICursor + { + private static readonly JsonSerializerSettings Settings = NuGetJsonSerialization.Settings; + private readonly string _path; + private readonly ILogger _logger; + + public FileCursor(string path, ILogger logger) + { + _path = path ?? throw new ArgumentNullException(nameof(path)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public Task GetAsync() + { + try + { + var jsonString = File.ReadAllText(_path); + var data = JsonConvert.DeserializeObject(jsonString, Settings); + _logger.LogDebug("Read cursor value {cursor:O} from {path}.", data.Value, _path); + return Task.FromResult(data.Value); + } + catch (Exception e) when (e is FileNotFoundException || e is JsonException) + { + return Task.FromResult(null); + } + } + + public Task SetAsync(DateTimeOffset value) + { + var data = new Data { Value = value }; + var jsonString = JsonConvert.SerializeObject(data); + File.WriteAllText(_path, jsonString); + _logger.LogDebug("Wrote cursor value {cursor:O} to {path}.", data.Value, _path); + return Task.CompletedTask; + } + + private class Data + { + [JsonProperty("value")] + public DateTimeOffset Value { get; set; } + } + } +} diff --git a/src/NuGet.Protocol.Catalog/ICatalogClient.cs b/src/NuGet.Protocol.Catalog/ICatalogClient.cs new file mode 100644 index 000000000..4ecbee386 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/ICatalogClient.cs @@ -0,0 +1,54 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Protocol.Catalog +{ + public interface ICatalogClient + { + /// + /// Get the catalog index at the provided URL. The catalog index URL should be discovered from the + /// service index. + /// + /// The catalog index URL. + /// The catalog index. + Task GetIndexAsync(string indexUrl); + + /// + /// Get the catalog page at the provided URL. The catalog page URL should be discovered from the catalog + /// index. + /// + /// The catalog page URL. + /// The catalog page. + Task GetPageAsync(string pageUrl); + + /// + /// Gets the catalog leaf at the provided URL. The catalog leaf URL should be discovered from a catalog page. + /// The type of the catalog leaf is automatically determined from the fetched document. + /// + /// The catalog leaf URL. + /// The catalog leaf. + Task GetLeafAsync(string leafUrl); + + /// + /// Gets the catalog leaf at the provided URL. The catalog leaf URL should be discovered from a catalog page. + /// The type of the catalog leaf must be a package delete. If the actual document is not a package delete, an + /// exception is thrown. + /// + /// The catalog leaf URL. + /// Thrown if the actual document is not a package delete. + /// The catalog leaf. + Task GetPackageDeleteLeafAsync(string leafUrl); + + /// + /// Gets the catalog leaf at the provided URL. The catalog leaf URL should be discovered from a catalog page. + /// The type of the catalog leaf must be package details. If the actual document is not package details, an + /// exception is thrown. + /// + /// The catalog leaf URL. + /// Thrown if the actual document is not package details. + /// The catalog leaf. + Task GetPackageDetailsLeafAsync(string leafUrl); + } +} \ No newline at end of file diff --git a/src/NuGet.Protocol.Catalog/ICatalogLeafProcessor.cs b/src/NuGet.Protocol.Catalog/ICatalogLeafProcessor.cs new file mode 100644 index 000000000..74b1f9733 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/ICatalogLeafProcessor.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Protocol.Catalog +{ + /// + /// An interface which allows custom processing of catalog leaves. This interface should be implemented when the + /// catalog leaf documents need to be downloaded and processed in chronological order. + /// + public interface ICatalogLeafProcessor + { + /// + /// Process a catalog leaf containing package details. This method should return false or throw an exception + /// if the catalog leaf cannot be processed. In this case, the will stop + /// processing items. Note that the same package ID/version combination can be passed to this multiple times, + /// for example due to an edit in the package metadata or due to a transient error and retry on the part of the + /// . + /// + /// The leaf document. + /// True, if the leaf was successfully processed. False, otherwise. + Task ProcessPackageDetailsAsync(PackageDetailsCatalogLeaf leaf); + + /// + /// Process a catalog leaf containing a package delete. This method should return false or throw an exception + /// if the catalog leaf cannot be processed. In this case, the will stop + /// processing items. Note that the same package ID/version combination can be passed to this multiple times, + /// for example due to a package being deleted again due to a transient error and retry on the part of the + /// . + /// + /// The leaf document. + /// True, if the leaf was successfully processed. False, otherwise. + Task ProcessPackageDeleteAsync(PackageDeleteCatalogLeaf leaf); + } +} diff --git a/src/NuGet.Protocol.Catalog/ICursor.cs b/src/NuGet.Protocol.Catalog/ICursor.cs new file mode 100644 index 000000000..241699b52 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/ICursor.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; + +namespace NuGet.Protocol.Catalog +{ + /// + /// An interface which allows reading and writing a cursor value. The value is up to what point in the catalog + /// has been successfully processed. The value is a catalog commit timestamp. + /// + public interface ICursor + { + /// + /// Get the value of the cursor. + /// + /// The cursor value. Null if the cursor has no value yet. + Task GetAsync(); + + /// + /// Set the value of the cursor. + /// + /// The new cursor value. + Task SetAsync(DateTimeOffset value); + } +} diff --git a/src/NuGet.Protocol.Catalog/ISimpleHttpClient.cs b/src/NuGet.Protocol.Catalog/ISimpleHttpClient.cs new file mode 100644 index 000000000..defba6236 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/ISimpleHttpClient.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Protocol.Catalog +{ + public interface ISimpleHttpClient + { + Task GetByteArrayAsync(string requestUri); + T DeserializeBytes(byte[] jsonBytes); + Task> DeserializeUrlAsync(string documentUrl); + } +} \ No newline at end of file diff --git a/src/NuGet.Protocol.Catalog/Models/AlternatePackage.cs b/src/NuGet.Protocol.Catalog/Models/AlternatePackage.cs new file mode 100644 index 000000000..d393d73a4 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/AlternatePackage.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + /// + /// Source: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#alternate-package + /// + public class AlternatePackage + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("@type")] + public string Type { get; set; } + + [JsonProperty("id")] + public string Id { get; set; } + + [JsonProperty("range")] + public string Range { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/BasePackageDependencyGroup.cs b/src/NuGet.Protocol.Catalog/Models/BasePackageDependencyGroup.cs new file mode 100644 index 000000000..a77a83572 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/BasePackageDependencyGroup.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + public abstract class BasePackageDependencyGroup where TDependency : PackageDependency + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("@type")] + public string Type { get; set; } + + [JsonProperty("dependencies")] + public List Dependencies { get; set; } + + [JsonProperty("targetFramework")] + public string TargetFramework { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/CatalogIndex.cs b/src/NuGet.Protocol.Catalog/Models/CatalogIndex.cs new file mode 100644 index 000000000..eb69b380a --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/CatalogIndex.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + public class CatalogIndex + { + [JsonProperty("commitTimeStamp")] + public DateTimeOffset CommitTimestamp { get; set; } + + [JsonProperty("count")] + public int Count { get; set; } + + [JsonProperty("items")] + public List Items { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/CatalogLeaf.cs b/src/NuGet.Protocol.Catalog/Models/CatalogLeaf.cs new file mode 100644 index 000000000..9f611d3e0 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/CatalogLeaf.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + public class CatalogLeaf : ICatalogLeafItem + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("@type")] + [JsonConverter(typeof(CatalogLeafTypeConverter))] + public CatalogLeafType Type { get; set; } + + [JsonProperty("catalog:commitId")] + public string CommitId { get; set; } + + [JsonProperty("catalog:commitTimeStamp")] + public DateTimeOffset CommitTimestamp { get; set; } + + [JsonProperty("id")] + public string PackageId { get; set; } + + [JsonProperty("published")] + public DateTimeOffset Published { get; set; } + + [JsonProperty("version")] + public string PackageVersion { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/CatalogLeafItem.cs b/src/NuGet.Protocol.Catalog/Models/CatalogLeafItem.cs new file mode 100644 index 000000000..d847af67a --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/CatalogLeafItem.cs @@ -0,0 +1,30 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + public class CatalogLeafItem : ICatalogLeafItem + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("@type")] + [JsonConverter(typeof(CatalogLeafItemTypeConverter))] + public CatalogLeafType Type { get; set; } + + [JsonProperty("commitId")] + public string CommitId { get; set; } + + [JsonProperty("commitTimeStamp")] + public DateTimeOffset CommitTimestamp { get; set; } + + [JsonProperty("nuget:id")] + public string PackageId { get; set; } + + [JsonProperty("nuget:version")] + public string PackageVersion { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/CatalogLeafType.cs b/src/NuGet.Protocol.Catalog/Models/CatalogLeafType.cs new file mode 100644 index 000000000..d23ef3d20 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/CatalogLeafType.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Protocol.Catalog +{ + public enum CatalogLeafType + { + PackageDetails = 1, + + PackageDelete = 2, + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/CatalogPage.cs b/src/NuGet.Protocol.Catalog/Models/CatalogPage.cs new file mode 100644 index 000000000..176fdb725 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/CatalogPage.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + public class CatalogPage + { + [JsonProperty("commitTimeStamp")] + public DateTimeOffset CommitTimestamp { get; set; } + + [JsonProperty("count")] + public int Count { get; set; } + + [JsonProperty("parent")] + public string Parent { get; set; } + + [JsonProperty("items")] + public List Items { get; set; } + + [JsonProperty("@context")] + public CatalogPageContext Context { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/CatalogPageContext.cs b/src/NuGet.Protocol.Catalog/Models/CatalogPageContext.cs new file mode 100644 index 000000000..f2028fbc9 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/CatalogPageContext.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + public class CatalogPageContext + { + [JsonProperty("@vocab")] + public string Vocab { get; set; } + + [JsonProperty("nuget")] + public string NuGet { get; set; } + + [JsonProperty("@items")] + public ContextTypeDescription Items { get; set; } + + [JsonProperty("parent")] + public ContextTypeDescription Parent { get; set; } + + [JsonProperty("commitTimeStamp")] + public ContextTypeDescription CommitTimestamp { get; set; } + + [JsonProperty("nuget:lastCreated")] + public ContextTypeDescription LastCreated { get; set; } + + [JsonProperty("nuget:lastEdited")] + public ContextTypeDescription LastEdited { get; set; } + + [JsonProperty("nuget:lastDeleted")] + public ContextTypeDescription LastDeleted { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/CatalogPageItem.cs b/src/NuGet.Protocol.Catalog/Models/CatalogPageItem.cs new file mode 100644 index 000000000..4677d5682 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/CatalogPageItem.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + public class CatalogPageItem + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("commitTimeStamp")] + public DateTimeOffset CommitTimestamp { get; set; } + + [JsonProperty("count")] + public int Count { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/ContextTypeDescription.cs b/src/NuGet.Protocol.Catalog/Models/ContextTypeDescription.cs new file mode 100644 index 000000000..adb4e801d --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/ContextTypeDescription.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + public class ContextTypeDescription + { + [JsonProperty("@id")] + public string Id { get; set; } + + [JsonProperty("@container")] + public string Container { get; set; } + + [JsonProperty("@type")] + public string Type { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/ICatalogLeafItem.cs b/src/NuGet.Protocol.Catalog/Models/ICatalogLeafItem.cs new file mode 100644 index 000000000..2c52ac455 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/ICatalogLeafItem.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Protocol.Catalog +{ + public interface ICatalogLeafItem + { + string CommitId { get; } + DateTimeOffset CommitTimestamp { get; } + string PackageId { get; } + string PackageVersion { get; } + CatalogLeafType Type { get; } + } +} \ No newline at end of file diff --git a/src/NuGet.Protocol.Catalog/Models/ModelExtensions.cs b/src/NuGet.Protocol.Catalog/Models/ModelExtensions.cs new file mode 100644 index 000000000..baaaef91c --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/ModelExtensions.cs @@ -0,0 +1,224 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using NuGet.Frameworks; +using NuGet.Packaging.Core; +using NuGet.Versioning; + +namespace NuGet.Protocol.Catalog +{ + /// + /// These are documented interpretations of values returned by the catalog API. + /// + public static class ModelExtensions + { + /// + /// Gets the leaves that lie within the provided commit timestamp bounds. The result is sorted by commit + /// timestamp, then package ID, then package version (SemVer order). + /// + /// + /// The exclusive lower time bound on . + /// The inclusive upper time bound on . + /// Only show the latest leaf concerning each package. + public static List GetLeavesInBounds( + this CatalogPage catalogPage, + DateTimeOffset minCommitTimestamp, + DateTimeOffset maxCommitTimestamp, + bool excludeRedundantLeaves) + { + var leaves = catalogPage + .Items + .Where(x => x.CommitTimestamp > minCommitTimestamp && x.CommitTimestamp <= maxCommitTimestamp) + .OrderBy(x => x.CommitTimestamp); + + if (excludeRedundantLeaves) + { + leaves = leaves + .GroupBy(x => new PackageIdentity(x.PackageId, x.ParsePackageVersion())) + .Select(x => x.Last()) + .OrderBy(x => x.CommitTimestamp); + } + + return leaves + .ThenBy(x => x.PackageId, StringComparer.OrdinalIgnoreCase) + .ThenBy(x => x.ParsePackageVersion()) + .ToList(); + } + + /// + /// Gets the pages that may have catalog leaves within the provided commit timestamp bounds. The result is + /// sorted by commit timestamp. + /// + /// The catalog index to fetch pages from. + /// The exclusive lower time bound on . + /// The inclusive upper time bound on . + public static List GetPagesInBounds( + this CatalogIndex catalogIndex, + DateTimeOffset minCommitTimestamp, + DateTimeOffset maxCommitTimestamp) + { + return catalogIndex + .GetPagesInBoundsLazy(minCommitTimestamp, maxCommitTimestamp) + .ToList(); + } + + private static IEnumerable GetPagesInBoundsLazy( + this CatalogIndex catalogIndex, + DateTimeOffset minCommitTimestamp, + DateTimeOffset maxCommitTimestamp) + { + // Filter out pages that fall entirely before the minimum commit timestamp and sort the remaining pages by + // commit timestamp. + var upperRange = catalogIndex + .Items + .Where(x => x.CommitTimestamp > minCommitTimestamp) + .OrderBy(x => x.CommitTimestamp); + + // Take pages from the sorted list until the commit timestamp goes past the maximum commit timestamp. This + // essentially LINQ's TakeWhile plus one more element. + foreach (var page in upperRange) + { + yield return page; + + if (page.CommitTimestamp > maxCommitTimestamp) + { + break; + } + } + } + + /// + /// Parse the package version as a . + /// + /// The catalog leaf. + /// The package version. + public static NuGetVersion ParsePackageVersion(this ICatalogLeafItem leaf) + { + return NuGetVersion.Parse(leaf.PackageVersion); + } + + /// + /// Parse the target framework as a . + /// + /// The package dependency group. + /// The framework. + public static NuGetFramework ParseTargetFramework(this PackageDependencyGroup packageDependencyGroup) + { + if (string.IsNullOrEmpty(packageDependencyGroup.TargetFramework)) + { + return NuGetFramework.AnyFramework; + } + + return NuGetFramework.Parse(packageDependencyGroup.TargetFramework); + } + + /// + /// Parse the version range as a . + /// + /// The package dependency. + /// The version range. + public static VersionRange ParseRange(this PackageDependency packageDependency) + { + // Server side treats invalid version ranges as empty strings. + // Source: https://github.com/NuGet/NuGet.Services.Metadata/blob/382c214c60993edfd7158bc6d223fafeebbc920c/src/Catalog/Helpers/NuGetVersionUtility.cs#L25-L34 + // Client side treats empty string version ranges as the "all" range. + // Source: https://github.com/NuGet/NuGet.Client/blob/849063018d8ee08625774a2dcd07ab84224dabb9/src/NuGet.Core/NuGet.Protocol/DependencyInfo/RegistrationUtility.cs#L20-L30 + // Example: https://api.nuget.org/v3/catalog0/data/2016.03.14.21.19.28/servicestack.extras.serilog.2.0.1.json + if (!VersionRange.TryParse(packageDependency.Range, out var parsed)) + { + return VersionRange.All; + } + + return parsed; + } + + /// + /// Determines if the provided catalog leaf is a package delete. + /// + /// The catalog leaf. + /// True if the catalog leaf represents a package delete. + public static bool IsPackageDelete(this ICatalogLeafItem leaf) + { + return leaf.Type == CatalogLeafType.PackageDelete; + } + + /// + /// Determines if the provided catalog leaf is contains package details. + /// + /// The catalog leaf. + /// True if the catalog leaf contains package details. + public static bool IsPackageDetails(this ICatalogLeafItem leaf) + { + return leaf.Type == CatalogLeafType.PackageDetails; + } + + /// + /// Determines if the provided package details leaf represents a listed package. + /// + /// The catalog leaf. + /// True if the package is listed. + public static bool IsListed(this PackageDetailsCatalogLeaf leaf) + { + if (leaf.Listed.HasValue) + { + return leaf.Listed.Value; + } + + // A published year of 1900 indicates that this package is unlisted, when the listed property itself is + // not present (legacy behavior). + // Example: https://api.nuget.org/v3/catalog0/data/2015.02.01.06.22.45/antixss.4.0.1.json + return leaf.Published.Year != 1900; + } + + /// + /// Determines if the provied package details leaf represents a SemVer 2.0.0 package. A package is considered + /// SemVer 2.0.0 if it's version is SemVer 2.0.0 or one of its dependency version ranges is SemVer 2.0.0. + /// + /// The catalog leaf. + /// True if the package is SemVer 2.0.0. + public static bool IsSemVer2(this PackageDetailsCatalogLeaf leaf) + { + var parsedPackageVersion = leaf.ParsePackageVersion(); + if (parsedPackageVersion.IsSemVer2) + { + return true; + } + + if (leaf.VerbatimVersion != null) + { + var parsedVerbatimVersion = NuGetVersion.Parse(leaf.VerbatimVersion); + if (parsedVerbatimVersion.IsSemVer2) + { + return true; + } + } + + if (leaf.DependencyGroups != null) + { + foreach (var dependencyGroup in leaf.DependencyGroups) + { + // Example: https://api.nuget.org/v3/catalog0/data/2018.10.28.07.42.42/mvcsitemapprovider.3.3.0-pre1.json + if (dependencyGroup.Dependencies == null) + { + continue; + } + + foreach (var dependency in dependencyGroup.Dependencies) + { + var versionRange = dependency.ParseRange(); + if ((versionRange.MaxVersion != null && versionRange.MaxVersion.IsSemVer2) + || (versionRange.MinVersion != null && versionRange.MinVersion.IsSemVer2)) + { + return true; + } + } + } + } + + return false; + } + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/PackageDeleteCatalogLeaf.cs b/src/NuGet.Protocol.Catalog/Models/PackageDeleteCatalogLeaf.cs new file mode 100644 index 000000000..de8c1fae8 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/PackageDeleteCatalogLeaf.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Protocol.Catalog +{ + public class PackageDeleteCatalogLeaf : CatalogLeaf + { + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/PackageDependency.cs b/src/NuGet.Protocol.Catalog/Models/PackageDependency.cs new file mode 100644 index 000000000..2dd169fb0 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/PackageDependency.cs @@ -0,0 +1,24 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NuGet.Protocol.Catalog.Serialization; + +namespace NuGet.Protocol.Catalog +{ + public class PackageDependency + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("@type")] + public string Type { get; set; } + + [JsonProperty("id")] + public string Id { get; set; } + + [JsonProperty("range")] + [JsonConverter(typeof(PackageDependencyRangeConverter))] + public string Range { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/PackageDependencyGroup.cs b/src/NuGet.Protocol.Catalog/Models/PackageDependencyGroup.cs new file mode 100644 index 000000000..808a01ac9 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/PackageDependencyGroup.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Protocol.Catalog +{ + public class PackageDependencyGroup : BasePackageDependencyGroup + { + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/PackageDeprecation.cs b/src/NuGet.Protocol.Catalog/Models/PackageDeprecation.cs new file mode 100644 index 000000000..bc9a11122 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/PackageDeprecation.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + /// + /// Source: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#package-deprecation + /// + public class PackageDeprecation + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("@type")] + public string Type { get; set; } + + [JsonProperty("alternatePackage")] + public AlternatePackage AlternatePackage { get; set; } + + [JsonProperty("message")] + public string Message { get; set; } + + [JsonProperty("reasons")] + public List Reasons { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/PackageDetailsCatalogLeaf.cs b/src/NuGet.Protocol.Catalog/Models/PackageDetailsCatalogLeaf.cs new file mode 100644 index 000000000..3c1d6e7e1 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/PackageDetailsCatalogLeaf.cs @@ -0,0 +1,100 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + public class PackageDetailsCatalogLeaf : CatalogLeaf + { + [JsonProperty("authors")] + public string Authors { get; set; } + + [JsonProperty("copyright")] + public string Copyright { get; set; } + + [JsonProperty("created")] + public DateTimeOffset Created { get; set; } + + [JsonProperty("lastEdited")] + public DateTimeOffset LastEdited { get; set; } + + [JsonProperty("dependencyGroups")] + public List DependencyGroups { get; set; } + + [JsonProperty("deprecation")] + public PackageDeprecation Deprecation { get; set; } + + [JsonProperty("description")] + public string Description { get; set; } + + [JsonProperty("iconUrl")] + public string IconUrl { get; set; } + + /// + /// Note that an old bug in the NuGet.org catalog had this wrong in some cases. + /// Example: https://api.nuget.org/v3/catalog0/data/2016.03.11.21.02.55/mvid.fody.2.json + /// + [JsonProperty("isPrerelease")] + public bool IsPrerelease { get; set; } + + [JsonProperty("language")] + public string Language { get; set; } + + [JsonProperty("licenseUrl")] + public string LicenseUrl { get; set; } + + [JsonProperty("listed")] + public bool? Listed { get; set; } + + [JsonProperty("minClientVersion")] + public string MinClientVersion { get; set; } + + [JsonProperty("packageHash")] + public string PackageHash { get; set; } + + [JsonProperty("packageHashAlgorithm")] + public string PackageHashAlgorithm { get; set; } + + [JsonProperty("packageSize")] + public long PackageSize { get; set; } + + [JsonProperty("packageTypes")] + public List PackageTypes { get; set; } + + [JsonProperty("projectUrl")] + public string ProjectUrl { get; set; } + + [JsonProperty("releaseNotes")] + public string ReleaseNotes { get; set; } + + [JsonProperty("requireLicenseAcceptance")] + public bool? RequireLicenseAcceptance { get; set; } + + [JsonProperty("summary")] + public string Summary { get; set; } + + [JsonProperty("tags")] + public List Tags { get; set; } + + [JsonProperty("title")] + public string Title { get; set; } + + [JsonProperty("verbatimVersion")] + public string VerbatimVersion { get; set; } + + [JsonProperty("licenseExpression")] + public string LicenseExpression { get; set; } + + [JsonProperty("licenseFile")] + public string LicenseFile { get; set; } + + [JsonProperty("iconFile")] + public string IconFile { get; set; } + + [JsonProperty("vulnerabilities")] + public List Vulnerabilities { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/PackageType.cs b/src/NuGet.Protocol.Catalog/Models/PackageType.cs new file mode 100644 index 000000000..897d74980 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/PackageType.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + /// + /// Source: https://docs.microsoft.com/en-us/nuget/api/catalog-resource#catalog-leaf + /// + public class PackageType + { + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("version")] + public string Version { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/Models/PackageVulnerability.cs b/src/NuGet.Protocol.Catalog/Models/PackageVulnerability.cs new file mode 100644 index 000000000..813c0d411 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Models/PackageVulnerability.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + public class PackageVulnerability + { + [JsonProperty("@id")] + public string Id { get; set; } + + [JsonProperty("@type")] + public string Type { get; set; } + + [JsonProperty("advisoryUrl")] + public string AdvisoryUrl { get; set; } + + [JsonProperty("severity")] + public string Severity { get; set; } + } +} diff --git a/src/NuGet.Protocol.Catalog/NuGet.Protocol.Catalog.csproj b/src/NuGet.Protocol.Catalog/NuGet.Protocol.Catalog.csproj new file mode 100644 index 000000000..c6127daf4 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/NuGet.Protocol.Catalog.csproj @@ -0,0 +1,118 @@ + + + + + + Debug + AnyCPU + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} + Library + Properties + NuGet.Protocol.Catalog + NuGet.Protocol.Catalog + v4.7.2 + 512 + true + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + NuGet.Protocol.Catalog + A .NET library for consuming the NuGet API's catalog resource. + .NET Foundation + https://github.com/NuGet/NuGet.Services.Metadata/blob/master/LICENSE + https://github.com/NuGet/NuGet.Services.Metadata + git + https://github.com/NuGet/NuGet.Services.Metadata + + + + + + + + + + 0.3.0 + runtime; build; native; contentfiles; analyzers + all + + + 2.2.0 + + + 9.0.1 + + + 4.8.0 + runtime; build; native; contentfiles; analyzers + all + + + 5.0.0-preview1.5707 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + + \ No newline at end of file diff --git a/src/NuGet.Protocol.Catalog/Properties/AssemblyInfo.cs b/src/NuGet.Protocol.Catalog/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..ace49ddfa --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Properties/AssemblyInfo.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Protocol.Catalog")] +[assembly: ComVisible(false)] +[assembly: Guid("d44c2e89-2d98-44bd-8712-8ccbe4e67c9c")] + +#if SIGNED_BUILD +[assembly: InternalsVisibleTo("NuGet.Protocol.Catalog.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] +#else +[assembly: InternalsVisibleTo("NuGet.Protocol.Catalog.Tests")] +#endif diff --git a/src/NuGet.Protocol.Catalog/ResponseAndResult.cs b/src/NuGet.Protocol.Catalog/ResponseAndResult.cs new file mode 100644 index 000000000..a3701957f --- /dev/null +++ b/src/NuGet.Protocol.Catalog/ResponseAndResult.cs @@ -0,0 +1,52 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net; +using System.Net.Http; + +namespace NuGet.Protocol.Catalog +{ + public class ResponseAndResult + { + public ResponseAndResult( + HttpMethod method, + string requestUri, + HttpStatusCode statusCode, + string reasonPhrase, + bool hasResult, + T result) + { + Method = method ?? throw new ArgumentNullException(nameof(method)); + RequestUri = requestUri ?? throw new ArgumentNullException(nameof(requestUri)); + StatusCode = statusCode; + ReasonPhrase = reasonPhrase ?? throw new ArgumentNullException(nameof(reasonPhrase)); + HasResult = hasResult; + Result = result; + } + + public HttpMethod Method { get; } + public string RequestUri { get; } + public HttpStatusCode StatusCode { get; } + public string ReasonPhrase { get; } + public bool HasResult { get; } + public T Result { get; } + + public T GetResultOrThrow() + { + if (!HasResult) + { + throw new SimpleHttpClientException( + $"The HTTP request failed.{Environment.NewLine}" + + $"Request: {Method} {RequestUri}{Environment.NewLine}" + + $"Response: {(int)StatusCode} {ReasonPhrase}", + Method, + RequestUri, + StatusCode, + ReasonPhrase); + } + + return Result; + } + } +} diff --git a/src/NuGet.Protocol.Catalog/Serialization/BaseCatalogLeafConverter.cs b/src/NuGet.Protocol.Catalog/Serialization/BaseCatalogLeafConverter.cs new file mode 100644 index 000000000..3913bbb8f --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Serialization/BaseCatalogLeafConverter.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + internal abstract class BaseCatalogLeafConverter : JsonConverter + { + private readonly IReadOnlyDictionary _fromType; + + public BaseCatalogLeafConverter(IReadOnlyDictionary fromType) + { + _fromType = fromType; + } + + public override bool CanConvert(Type objectType) + { + return objectType == typeof(CatalogLeafType); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + string output; + if (_fromType.TryGetValue((CatalogLeafType)value, out output)) + { + writer.WriteValue(output); + return; + } + + throw new NotSupportedException($"The catalog leaf type '{value}' is not supported."); + } + } +} diff --git a/src/NuGet.Protocol.Catalog/Serialization/CatalogLeafItemTypeConverter.cs b/src/NuGet.Protocol.Catalog/Serialization/CatalogLeafItemTypeConverter.cs new file mode 100644 index 000000000..85dac704c --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Serialization/CatalogLeafItemTypeConverter.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + internal class CatalogLeafItemTypeConverter : BaseCatalogLeafConverter + { + private static readonly Dictionary FromType = new Dictionary + { + { CatalogLeafType.PackageDelete, "nuget:PackageDelete" }, + { CatalogLeafType.PackageDetails, "nuget:PackageDetails" }, + }; + + private static readonly Dictionary FromString = FromType + .ToDictionary(x => x.Value, x => x.Key); + + public CatalogLeafItemTypeConverter() : base(FromType) + { + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + string stringValue = reader.Value as string; + if (stringValue != null) + { + CatalogLeafType output; + if (FromString.TryGetValue(stringValue, out output)) + { + return output; + } + } + + throw new JsonSerializationException($"Unexpected value for a {nameof(CatalogLeafType)}."); + } + } +} diff --git a/src/NuGet.Protocol.Catalog/Serialization/CatalogLeafTypeConverter.cs b/src/NuGet.Protocol.Catalog/Serialization/CatalogLeafTypeConverter.cs new file mode 100644 index 000000000..d8bbc576d --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Serialization/CatalogLeafTypeConverter.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + internal class CatalogLeafTypeConverter : BaseCatalogLeafConverter + { + private static readonly Dictionary FromType = new Dictionary + { + { CatalogLeafType.PackageDelete, "PackageDelete" }, + { CatalogLeafType.PackageDetails, "PackageDetails" }, + }; + + private static readonly Dictionary FromString = FromType + .ToDictionary(x => x.Value, x => x.Key); + + public CatalogLeafTypeConverter() : base(FromType) + { + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + List types; + if (reader.TokenType == JsonToken.StartArray) + { + types = serializer.Deserialize>(reader); + } + else + { + types = new List { reader.Value }; + } + + foreach (var type in types.OfType()) + { + CatalogLeafType output; + if (FromString.TryGetValue(type, out output)) + { + return output; + } + } + + throw new JsonSerializationException($"Unexpected value for a {nameof(CatalogLeafType)}."); + } + } +} diff --git a/src/NuGet.Protocol.Catalog/Serialization/NuGetJsonSerialization.cs b/src/NuGet.Protocol.Catalog/Serialization/NuGetJsonSerialization.cs new file mode 100644 index 000000000..a2e7e8d37 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Serialization/NuGetJsonSerialization.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + public static class NuGetJsonSerialization + { + public static JsonSerializer Serializer => JsonSerializer.Create(Settings); + + public static JsonSerializerSettings Settings => new JsonSerializerSettings + { + DateTimeZoneHandling = DateTimeZoneHandling.Utc, + DateParseHandling = DateParseHandling.DateTimeOffset, + NullValueHandling = NullValueHandling.Ignore, + }; + } +} diff --git a/src/NuGet.Protocol.Catalog/Serialization/PackageDependencyRangeConverter.cs b/src/NuGet.Protocol.Catalog/Serialization/PackageDependencyRangeConverter.cs new file mode 100644 index 000000000..cf092738b --- /dev/null +++ b/src/NuGet.Protocol.Catalog/Serialization/PackageDependencyRangeConverter.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog.Serialization +{ + public class PackageDependencyRangeConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(string); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.StartArray) + { + // There are some quirky packages with arrays of dependency version ranges. In this case, we take the + // first element. + // Example: https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json + var array = serializer.Deserialize(reader); + return array.FirstOrDefault(); + } + + return serializer.Deserialize(reader); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + serializer.Serialize(writer, value); + } + } +} diff --git a/src/NuGet.Protocol.Catalog/SimpleHttpClient.cs b/src/NuGet.Protocol.Catalog/SimpleHttpClient.cs new file mode 100644 index 000000000..789f9e9a7 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/SimpleHttpClient.cs @@ -0,0 +1,75 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Catalog +{ + public class SimpleHttpClient : ISimpleHttpClient + { + private static readonly JsonSerializer JsonSerializer = NuGetJsonSerialization.Serializer; + private readonly HttpClient _httpClient; + private readonly ILogger _logger; + + public SimpleHttpClient(HttpClient httpClient, ILogger logger) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task GetByteArrayAsync(string requestUri) + { + return await _httpClient.GetByteArrayAsync(requestUri); + } + + public T DeserializeBytes(byte[] jsonBytes) + { + using (var stream = new MemoryStream(jsonBytes)) + using (var textReader = new StreamReader(stream)) + using (var jsonReader = new JsonTextReader(textReader)) + { + return JsonSerializer.Deserialize(jsonReader); + } + } + + public async Task> DeserializeUrlAsync(string documentUrl) + { + _logger.LogDebug("Downloading {documentUrl} as a stream.", documentUrl); + + using (var response = await _httpClient.GetAsync( + documentUrl, + HttpCompletionOption.ResponseHeadersRead)) + { + if (response.StatusCode != HttpStatusCode.OK) + { + return new ResponseAndResult( + HttpMethod.Get, + documentUrl, + response.StatusCode, + response.ReasonPhrase, + hasResult: false, + result: default(T)); + } + + using (var stream = await response.Content.ReadAsStreamAsync()) + using (var textReader = new StreamReader(stream)) + using (var jsonReader = new JsonTextReader(textReader)) + { + return new ResponseAndResult( + HttpMethod.Get, + documentUrl, + response.StatusCode, + response.ReasonPhrase, + hasResult: true, + result: JsonSerializer.Deserialize(jsonReader)); + } + } + } + } +} diff --git a/src/NuGet.Protocol.Catalog/SimpleHttpClientException.cs b/src/NuGet.Protocol.Catalog/SimpleHttpClientException.cs new file mode 100644 index 000000000..396dd6ed8 --- /dev/null +++ b/src/NuGet.Protocol.Catalog/SimpleHttpClientException.cs @@ -0,0 +1,30 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net; +using System.Net.Http; + +namespace NuGet.Protocol.Catalog +{ + public class SimpleHttpClientException : Exception + { + public SimpleHttpClientException( + string message, + HttpMethod method, + string requestUri, + HttpStatusCode statusCode, + string reasonPhrase) : base(message) + { + Method = method ?? throw new ArgumentNullException(nameof(method)); + RequestUri = requestUri ?? throw new ArgumentNullException(nameof(requestUri)); + StatusCode = statusCode; + ReasonPhrase = reasonPhrase ?? throw new ArgumentNullException(nameof(reasonPhrase)); + } + + public HttpMethod Method { get; } + public string RequestUri { get; } + public HttpStatusCode StatusCode { get; } + public string ReasonPhrase { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Analysis/DescriptionCustomAnalyzer.cs b/src/NuGet.Services.AzureSearch/Analysis/DescriptionCustomAnalyzer.cs new file mode 100644 index 000000000..93de702aa --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Analysis/DescriptionCustomAnalyzer.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Support for NuGet style description analysis. This splits tokens + /// on non alpha-numeric characters, splits tokens on camel casing, + /// lower cases tokens, and then removes stopwords from tokens. + /// + public static class DescriptionAnalyzer + { + public const string Name = "nuget_description_analyzer"; + + public static readonly CustomAnalyzer Instance = new CustomAnalyzer( + Name, + PackageIdCustomTokenizer.Name, + new List + { + IdentifierCustomTokenFilter.Name, + TokenFilterName.Stopwords, + TokenFilterName.Lowercase, + TruncateCustomTokenFilter.Name, + }); + } +} diff --git a/src/NuGet.Services.AzureSearch/Analysis/ExactMatchCustomAnalyzer.cs b/src/NuGet.Services.AzureSearch/Analysis/ExactMatchCustomAnalyzer.cs new file mode 100644 index 000000000..bcd7e4c20 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Analysis/ExactMatchCustomAnalyzer.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Support for case-insensitive exact matching on a field + /// in an Azure Search index. + /// + public static class ExactMatchCustomAnalyzer + { + public const string Name = "nuget_exact_match_analyzer"; + + public static readonly CustomAnalyzer Instance = new CustomAnalyzer( + Name, + TokenizerName.Keyword, + new List + { + TokenFilterName.Lowercase + }); + } +} diff --git a/src/NuGet.Services.AzureSearch/Analysis/IdentifierCustomTokenFilter.cs b/src/NuGet.Services.AzureSearch/Analysis/IdentifierCustomTokenFilter.cs new file mode 100644 index 000000000..b21d784ab --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Analysis/IdentifierCustomTokenFilter.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Splits tokens on camel casing and non alpha-numeric characters. + /// This does not consume the original token. For example, "Foo2Bar.Baz" + /// becomes "Foo", "2", "Bar", "Baz", and "Foo2Bar.Baz". + /// + public static class IdentifierCustomTokenFilter + { + public const string Name = "nuget_id_filter"; + + public static WordDelimiterTokenFilter Instance = new WordDelimiterTokenFilter( + Name, + splitOnCaseChange: true, + preserveOriginal: true); + } +} diff --git a/src/NuGet.Services.AzureSearch/Analysis/PackageIdCustomAnalyzer.cs b/src/NuGet.Services.AzureSearch/Analysis/PackageIdCustomAnalyzer.cs new file mode 100644 index 000000000..2ac6c4d9d --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Analysis/PackageIdCustomAnalyzer.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Support for NuGet style identifier analysis. Splits tokens + /// on non alpha-numeric characters and camel casing, and lower + /// cases tokens. + /// + public static class PackageIdCustomAnalyzer + { + public const string Name = "nuget_package_id_analyzer"; + + public static readonly CustomAnalyzer Instance = new CustomAnalyzer( + Name, + PackageIdCustomTokenizer.Name, + new List + { + IdentifierCustomTokenFilter.Name, + TokenFilterName.Lowercase, + TruncateCustomTokenFilter.Name, + }); + } +} diff --git a/src/NuGet.Services.AzureSearch/Analysis/PackageIdCustomTokenizer.cs b/src/NuGet.Services.AzureSearch/Analysis/PackageIdCustomTokenizer.cs new file mode 100644 index 000000000..1a45fdd9a --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Analysis/PackageIdCustomTokenizer.cs @@ -0,0 +1,17 @@ +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Splits tokens that on a set of symbols and whitespace. + /// For example, "Foo.Bar Baz" becomes "Foo", "Bar", and "Baz". + /// + public static class PackageIdCustomTokenizer + { + public const string Name = "nuget_package_id_tokenizer"; + + public static readonly PatternTokenizer Instance = new PatternTokenizer( + Name, + @"[.\-_,;:'*#!~+()\[\]{}\s]"); + } +} diff --git a/src/NuGet.Services.AzureSearch/Analysis/TagsCustomAnalyzer.cs b/src/NuGet.Services.AzureSearch/Analysis/TagsCustomAnalyzer.cs new file mode 100644 index 000000000..f8c796c95 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Analysis/TagsCustomAnalyzer.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Support for tag case-insensitive exact matching on a field with trimming + /// in an Azure Search index. + /// + /// Note: will not split on special characters like -,_,$,etc. This is important + /// and allows developers to hyphenate their tags. It will trim excess whitespace + /// from the end of each tag. + /// + /// Tokenization will also exclude duplicates from the indexing process. + /// + public static class TagsCustomAnalyzer + { + public const string Name = "nuget_tags_analyzer"; + + public static readonly CustomAnalyzer Instance = new CustomAnalyzer( + Name, + TokenizerName.Keyword, + new List + { + TokenFilterName.Lowercase, + TokenFilterName.Trim, + TokenFilterName.Unique + }); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Analysis/TruncateCustomTokenFilter.cs b/src/NuGet.Services.AzureSearch/Analysis/TruncateCustomTokenFilter.cs new file mode 100644 index 000000000..ba46a7f7a --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Analysis/TruncateCustomTokenFilter.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Truncates tokens to 300 characters or less. This is necessary as Azure Search's + /// defaults to 10 characters. + /// + public static class TruncateCustomTokenFilter + { + public const string Name = "nuget_truncate_filter"; + + public static TruncateTokenFilter Instance = new TruncateTokenFilter( + Name, + length: 300); + } +} diff --git a/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/Auxiliary2AzureSearchCommand.cs b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/Auxiliary2AzureSearchCommand.cs new file mode 100644 index 000000000..320d10a9b --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/Auxiliary2AzureSearchCommand.cs @@ -0,0 +1,56 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch +{ + public class Auxiliary2AzureSearchCommand : IAzureSearchCommand + { + private readonly IAzureSearchCommand[] _commands; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + + public Auxiliary2AzureSearchCommand( + IAzureSearchCommand updateVerifiedPackagesCommand, + IAzureSearchCommand updateDownloadsCommand, + IAzureSearchCommand updateOwnersCommand, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _commands = new[] + { + updateVerifiedPackagesCommand ?? throw new ArgumentNullException(nameof(updateVerifiedPackagesCommand)), + updateDownloadsCommand ?? throw new ArgumentNullException(nameof(updateDownloadsCommand)), + updateOwnersCommand ?? throw new ArgumentNullException(nameof(updateOwnersCommand)), + }; + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ExecuteAsync() + { + var stopwatch = Stopwatch.StartNew(); + var outcome = JobOutcome.Failure; + try + { + foreach (var command in _commands) + { + _logger.LogInformation("Starting {CommandName}...", command.GetType().Name); + await command.ExecuteAsync(); + _logger.LogInformation("Completed {CommandName}.", command.GetType().Name); + } + + outcome = JobOutcome.Success; + } + finally + { + stopwatch.Stop(); + _telemetryService.TrackAuxiliary2AzureSearchCompleted(outcome, stopwatch.Elapsed); + } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/Auxiliary2AzureSearchConfiguration.cs b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/Auxiliary2AzureSearchConfiguration.cs new file mode 100644 index 000000000..ca53293e3 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/Auxiliary2AzureSearchConfiguration.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Services.AzureSearch.AuxiliaryFiles; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch +{ + public class Auxiliary2AzureSearchConfiguration : AzureSearchJobConfiguration, IAuxiliaryDataStorageConfiguration + { + public string AuxiliaryDataStorageConnectionString { get; set; } + public string AuxiliaryDataStorageContainer { get; set; } + public string AuxiliaryDataStorageDownloadsPath { get; set; } + public string AuxiliaryDataStorageExcludedPackagesPath { get; } + public string AuxiliaryDataStorageVerifiedPackagesPath { get; set; } + public TimeSpan MinPushPeriod { get; set; } + public int MaxDownloadCountDecreases { get; set; } = 15000; + } +} diff --git a/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/DataSetComparer.cs b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/DataSetComparer.cs new file mode 100644 index 000000000..6536a56ec --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/DataSetComparer.cs @@ -0,0 +1,146 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Microsoft.Extensions.Logging; +using NuGet.Services.AzureSearch.AuxiliaryFiles; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch +{ + public class DataSetComparer : IDataSetComparer + { + private static readonly string[] EmptyStringArray = new string[0]; + + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + + public DataSetComparer( + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public SortedDictionary CompareOwners( + SortedDictionary> oldData, + SortedDictionary> newData) + { + if (oldData.Comparer != StringComparer.OrdinalIgnoreCase) + { + throw new ArgumentException("The old data should have a case-insensitive comparer.", nameof(oldData)); + } + + if (newData.Comparer != StringComparer.OrdinalIgnoreCase) + { + throw new ArgumentException("The new data should have a case-insensitive comparer.", nameof(newData)); + } + + // Use ordinal comparison to allow username case changes to flow through. + var stopwatch = Stopwatch.StartNew(); + var result = CompareData( + oldData, + newData, + "package ID", + "owners", + StringComparer.Ordinal); + + stopwatch.Stop(); + _telemetryService.TrackOwnerSetComparison(oldData.Count, newData.Count, result.Count, stopwatch.Elapsed); + + return result; + } + + public SortedDictionary ComparePopularityTransfers( + PopularityTransferData oldData, + PopularityTransferData newData) + { + // Ignore case changes in popularity transfers. + var stopwatch = Stopwatch.StartNew(); + var result = CompareData( + oldData, + newData, + "package ID", + "popularity transfers", + StringComparer.OrdinalIgnoreCase); + + stopwatch.Stop(); + _telemetryService.TrackPopularityTransfersSetComparison(oldData.Count, newData.Count, result.Count, stopwatch.Elapsed); + + return result; + } + + private SortedDictionary CompareData( + IReadOnlyDictionary> oldData, + IReadOnlyDictionary> newData, + string keyName, + string valuesName, + StringComparer valuesComparer) + { + // We use a very simplistic algorithm here. Perform one pass on the new data to find the added or changed + // values. Then perform a second pass on the old data to find removed keys. We can optimize + // this later if necessary. + // + // On the "changed" case, we emit all of the values instead of the delta. This is because Azure Search + // does not have a way to append or remove a specific item from a field that is an array. + // The entire new array needs to be provided. + var result = new SortedDictionary(StringComparer.OrdinalIgnoreCase); + + // First pass: find added or changed sets. + foreach (var pair in newData) + { + var key = pair.Key; + var newValues = pair.Value; + if (!oldData.TryGetValue(key, out var oldValues)) + { + // ADDED: The key does not exist in the old data, which means the key was added. + result.Add(key, newValues.ToArray()); + _logger.LogInformation( + $"The {keyName} {{Key}} has been added, with {{AddedCount}} {valuesName}.", + key, + newValues.Count); + } + else + { + // The key exists in the old data. We need to check if the values set has changed. + var removedValues = oldValues.Except(newValues, valuesComparer).ToList(); + var addedValues = newValues.Except(oldValues, valuesComparer).ToList(); + + if (removedValues.Any() || addedValues.Any()) + { + // CHANGED: The values set has changed. + result.Add(key, newValues.ToArray()); + _logger.LogInformation( + $"The {keyName} {{Key}} {valuesName} have changed, with {{RemovedCount}} {valuesName} removed and " + + $"{{AddedCount}} {valuesName} added.", + key, + removedValues.Count, + addedValues.Count); + } + } + } + + // Second pass: find removed sets. + foreach (var pair in oldData) + { + var key = pair.Key; + var oldValues = pair.Value; + + if (!newData.TryGetValue(key, out var newValues)) + { + // REMOVED: The key does not exist in the new data, which means the key was removed. + result.Add(key, EmptyStringArray); + _logger.LogInformation( + $"The {keyName} {{Key}} has been removed, with {{RemovedCount}} {valuesName}", + key, + oldValues.Count); + } + } + + return result; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/DownloadSetComparer.cs b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/DownloadSetComparer.cs new file mode 100644 index 000000000..f81962586 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/DownloadSetComparer.cs @@ -0,0 +1,128 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Services.AzureSearch.AuxiliaryFiles; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch +{ + public class DownloadSetComparer : IDownloadSetComparer + { + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public DownloadSetComparer( + IAzureSearchTelemetryService telemetryService, + IOptionsSnapshot options, + ILogger logger) + { + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public SortedDictionary Compare( + DownloadData oldData, + DownloadData newData) + { + if (newData.Count == 0) + { + throw new InvalidOperationException("The new data should not be empty."); + } + + var stopwatch = Stopwatch.StartNew(); + + // We use a very simplistic algorithm here. Find the union of both ID sets and compare each download count. + var uniqueIds = new HashSet( + oldData.Keys.Concat(newData.Keys), + StringComparer.OrdinalIgnoreCase); + + _logger.LogInformation( + "There are {OldCount} IDs in the old data, {NewCount} IDs in the new data, and {TotalCount} IDs in total.", + oldData.Count, + newData.Count, + uniqueIds.Count); + + var result = new SortedDictionary(StringComparer.OrdinalIgnoreCase); + var decreaseCount = 0; + foreach (var id in uniqueIds) + { + // Detect download count decreases and emit a metric. This is not necessarily wrong because there have + // been times that we manually delete spoofed download counts. + DetectDownloadCountDecreases(oldData, newData, id, ref decreaseCount); + + var oldCount = oldData.GetDownloadCount(id); + var newCount = newData.GetDownloadCount(id); + if (oldCount != newCount) + { + result.Add(id, newCount); + } + } + + _logger.LogInformation("There are {Count} package IDs with download count changes.", result.Count); + _logger.LogInformation("There are {Count} package versions with download count decreases.", decreaseCount); + + if (decreaseCount > _options.Value.MaxDownloadCountDecreases) + { + throw new InvalidOperationException("Too many download count decreases are occurring."); + } + + stopwatch.Stop(); + _telemetryService.TrackDownloadSetComparison(oldData.Count, newData.Count, result.Count, stopwatch.Elapsed); + + return result; + } + + private void DetectDownloadCountDecreases(DownloadData oldData, DownloadData newData, string id, ref int decreaseCount) + { + var oldHasId = oldData.TryGetValue(id, out var oldDownloads); + if (!oldHasId) + { + oldDownloads = new DownloadByVersionData(); + } + + var newHasId = newData.TryGetValue(id, out var newDownloads); + if (!newHasId) + { + newDownloads = new DownloadByVersionData(); + } + + var uniqueVersions = new HashSet( + oldDownloads.Keys.Concat(newDownloads.Keys), + StringComparer.OrdinalIgnoreCase); + + foreach (var version in uniqueVersions) + { + var oldHasVersion = oldDownloads.TryGetValue(version, out var oldCount); + var newHasVersion = newDownloads.TryGetValue(version, out var newCount); + + if (newCount < oldCount) + { + decreaseCount++; + + // Don't emit too many telemetry events. At a certain point the detail provided by additional events + // doesn't help investigation and can overwhelm Application Insights. + if (decreaseCount <= _options.Value.MaxDownloadCountDecreases) + { + _telemetryService.TrackDownloadCountDecrease( + id, + version, + oldHasId, + oldHasVersion, + oldCount, + newHasId, + newHasVersion, + newCount); + } + } + } + } + } +} + diff --git a/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/IDataSetComparer.cs b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/IDataSetComparer.cs new file mode 100644 index 000000000..35287712c --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/IDataSetComparer.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using NuGet.Services.AzureSearch.AuxiliaryFiles; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch +{ + /// + /// Used to compare two sets of data to determine the changes. + /// + public interface IDataSetComparer + { + /// + /// Compares two sets of owners to determine the package IDs that have changed. The returned dictionary + /// contains owners that were added, removed, and changed. For the "added" and "changed" cases, the owner set + /// is the new data. For the "removed" case, the set is empty. The key of the returned dictionary is the package + /// ID and the value is the set of owners. The returned set of owners is an array because this is the type used + /// on the Azure Search document models. + /// + /// The old owner information, typically from storage. + /// The new owner information, typically from gallery DB. + SortedDictionary CompareOwners( + SortedDictionary> oldData, + SortedDictionary> newData); + + /// + /// Compares two sets of popularity transfers to determine changes. The two inputs are maps of package IDs that transfer + /// popularity away to package IDs that receive the popularity. The returned dictionary is subset of these inputs that + /// were added, removed, or changed. For the "added" and "changed" cases, the popularity transfer set is the new data. + /// For the "removed" case, the set is empty. + /// + /// The old popularity transfers, typically from storage. + /// The new popularity transfers, typically from gallery DB. + SortedDictionary ComparePopularityTransfers( + PopularityTransferData oldData, + PopularityTransferData newData); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/IDownloadSetComparer.cs b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/IDownloadSetComparer.cs new file mode 100644 index 000000000..7fccdeff3 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/IDownloadSetComparer.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using NuGet.Services.AzureSearch.AuxiliaryFiles; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch +{ + public interface IDownloadSetComparer + { + /// + /// Compares the old download count data with the new download count data and returns the changes. + /// + /// The old data. + /// The new data. + /// The changes where the key is the package ID and the value is the new download count. + SortedDictionary Compare(DownloadData oldData, DownloadData newData); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/UpdateDownloadsCommand.cs b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/UpdateDownloadsCommand.cs new file mode 100644 index 000000000..e6391711b --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/UpdateDownloadsCommand.cs @@ -0,0 +1,437 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Packaging; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.AzureSearch.Wrappers; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Versioning; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch +{ + public class UpdateDownloadsCommand : IAzureSearchCommand + { + /// + /// A package ID can result in one document per search filter if the there is a version that applies to each + /// of the filters. The simplest such case is a prerelease, SemVer 1.0.0 package version like 1.0.0-beta. This + /// version applies to all package filters. + /// + private static readonly int MaxDocumentsPerId = Enum.GetValues(typeof(SearchFilters)).Length; + + private readonly IAuxiliaryFileClient _auxiliaryFileClient; + private readonly IDatabaseAuxiliaryDataFetcher _databaseFetcher; + private readonly IDownloadDataClient _downloadDataClient; + private readonly IDownloadSetComparer _downloadSetComparer; + private readonly IDownloadTransferrer _downloadTransferrer; + private readonly IPopularityTransferDataClient _popularityTransferDataClient; + private readonly ISearchDocumentBuilder _searchDocumentBuilder; + private readonly ISearchIndexActionBuilder _indexActionBuilder; + private readonly Func _batchPusherFactory; + private readonly ISystemTime _systemTime; + private readonly IFeatureFlagService _featureFlags; + private readonly IOptionsSnapshot _options; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + private readonly StringCache _stringCache; + + public UpdateDownloadsCommand( + IAuxiliaryFileClient auxiliaryFileClient, + IDatabaseAuxiliaryDataFetcher databaseFetcher, + IDownloadDataClient downloadDataClient, + IDownloadSetComparer downloadSetComparer, + IDownloadTransferrer downloadTransferrer, + IPopularityTransferDataClient popularityTransferDataClient, + ISearchDocumentBuilder searchDocumentBuilder, + ISearchIndexActionBuilder indexActionBuilder, + Func batchPusherFactory, + ISystemTime systemTime, + IFeatureFlagService featureFlags, + IOptionsSnapshot options, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _auxiliaryFileClient = auxiliaryFileClient ?? throw new ArgumentException(nameof(auxiliaryFileClient)); + _databaseFetcher = databaseFetcher ?? throw new ArgumentNullException(nameof(databaseFetcher)); + _downloadDataClient = downloadDataClient ?? throw new ArgumentNullException(nameof(downloadDataClient)); + _downloadSetComparer = downloadSetComparer ?? throw new ArgumentNullException(nameof(downloadSetComparer)); + _downloadTransferrer = downloadTransferrer ?? throw new ArgumentNullException(nameof(downloadTransferrer)); + _popularityTransferDataClient = popularityTransferDataClient ?? throw new ArgumentNullException(nameof(popularityTransferDataClient)); + _searchDocumentBuilder = searchDocumentBuilder ?? throw new ArgumentNullException(nameof(searchDocumentBuilder)); + _indexActionBuilder = indexActionBuilder ?? throw new ArgumentNullException(nameof(indexActionBuilder)); + _batchPusherFactory = batchPusherFactory ?? throw new ArgumentNullException(nameof(batchPusherFactory)); + _systemTime = systemTime ?? throw new ArgumentNullException(nameof(systemTime)); + _featureFlags = featureFlags ?? throw new ArgumentNullException(nameof(featureFlags)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _stringCache = new StringCache(); + + if (_options.Value.MaxConcurrentBatches <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(options), + $"The {nameof(AzureSearchJobConfiguration.MaxConcurrentBatches)} must be greater than zero."); + } + + if (_options.Value.MaxConcurrentVersionListWriters <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(options), + $"The {nameof(AzureSearchJobConfiguration.MaxConcurrentVersionListWriters)} must be greater than zero."); + } + } + + public async Task ExecuteAsync() + { + var stopwatch = Stopwatch.StartNew(); + var outcome = JobOutcome.Failure; + try + { + outcome = await PushIndexChangesAsync() ? JobOutcome.Success : JobOutcome.NoOp; + } + finally + { + stopwatch.Stop(); + _telemetryService.TrackUpdateDownloadsCompleted(outcome, stopwatch.Elapsed); + } + } + + private async Task PushIndexChangesAsync() + { + // The "old" data in this case is the download count data that was last indexed by this job (or + // initialized by Db2AzureSearch). + _logger.LogInformation("Fetching old download count data from blob storage."); + var oldResult = await _downloadDataClient.ReadLatestIndexedAsync( + AccessConditionWrapper.GenerateEmptyCondition(), + _stringCache); + + // The "new" data in this case is from the statistics pipeline. + _logger.LogInformation("Fetching new download count data from blob storage."); + var newData = await _auxiliaryFileClient.LoadDownloadDataAsync(); + + _logger.LogInformation("Removing invalid IDs and versions from the old downloads data."); + CleanDownloadData(oldResult.Data); + + _logger.LogInformation("Removing invalid IDs and versions from the new downloads data."); + CleanDownloadData(newData); + + _logger.LogInformation("Detecting download count changes."); + var changes = _downloadSetComparer.Compare(oldResult.Data, newData); + _logger.LogInformation("{Count} package IDs have download count changes.", changes.Count); + + // The "old" data is the popularity transfers data that was last indexed by this job (or + // initialized by Db2AzureSearch). + _logger.LogInformation("Fetching old popularity transfer data from blob storage."); + var oldTransfers = await _popularityTransferDataClient.ReadLatestIndexedAsync( + AccessConditionWrapper.GenerateEmptyCondition(), + _stringCache); + + // The "new" data is the latest popularity transfers data from the database. + _logger.LogInformation("Fetching new popularity transfer data from database."); + var newTransfers = await GetPopularityTransfersAsync(); + + _logger.LogInformation("Applying download transfers to download changes."); + ApplyDownloadTransfers( + newData, + oldTransfers.Data, + newTransfers, + changes); + + var idBag = new ConcurrentBag(changes.Keys); + _logger.LogInformation("{Count} package IDs need to be updated.", idBag.Count); + + if (!changes.Any()) + { + return false; + } + + _logger.LogInformation( + "Starting {Count} workers pushing download count changes to Azure Search.", + _options.Value.MaxConcurrentBatches); + await ParallelAsync.Repeat( + () => WorkAndRetryAsync(idBag, changes), + _options.Value.MaxConcurrentBatches); + _logger.LogInformation("All of the download count changes have been pushed to Azure Search."); + + _logger.LogInformation("Uploading the new download count data to blob storage."); + await _downloadDataClient.ReplaceLatestIndexedAsync(newData, oldResult.Metadata.GetIfMatchCondition()); + + _logger.LogInformation("Uploading the new popularity transfer data to blob storage."); + await _popularityTransferDataClient.ReplaceLatestIndexedAsync( + newTransfers, + oldTransfers.Metadata.GetIfMatchCondition()); + return true; + } + + private async Task GetPopularityTransfersAsync() + { + if (!_options.Value.EnablePopularityTransfers) + { + _logger.LogWarning( + "Popularity transfers are disabled. Popularity transfers will be ignored."); + return new PopularityTransferData(); + } + + if (!_featureFlags.IsPopularityTransferEnabled()) + { + _logger.LogWarning( + "Popularity transfers feature flag is disabled. " + + "All popularity transfers will be removed."); + return new PopularityTransferData(); + } + + return await _databaseFetcher.GetPopularityTransfersAsync(); + } + + private void ApplyDownloadTransfers( + DownloadData newData, + PopularityTransferData oldTransfers, + PopularityTransferData newTransfers, + SortedDictionary downloadChanges) + { + _logger.LogInformation("Finding download changes from popularity transfers."); + var transferChanges = _downloadTransferrer.UpdateDownloadTransfers( + newData, + downloadChanges, + oldTransfers, + newTransfers); + + _logger.LogInformation( + "{Count} package IDs have download count changes from popularity transfers.", + transferChanges.Count); + + // Apply the transfer changes to the overall download changes. + foreach (var transferChange in transferChanges) + { + downloadChanges[transferChange.Key] = transferChange.Value; + } + } + + private async Task WorkAndRetryAsync(ConcurrentBag idBag, SortedDictionary changes) + { + BatchPusherResult result; + var attempt = 0; + const int maxAttempts = 3; + do + { + attempt++; + result = await WorkAsync(idBag, changes); + + // Retry any failed package IDs. + if (result.FailedPackageIds.Any() && attempt < maxAttempts) + { + _logger.LogWarning("Attempt {Attempt} did not fully succeed. Retrying {Count} package IDs.", attempt, result.FailedPackageIds.Count); + foreach (var id in result.FailedPackageIds) + { + idBag.Add(id); + } + } + } + while (!result.Success && attempt < maxAttempts); + + result.EnsureSuccess(); + } + + private async Task WorkAsync(ConcurrentBag idBag, SortedDictionary changes) + { + // Perform two batching mechanisms: + // + // 1. Group package IDs into batches so version lists can be fetched in parallel. + // 2. Group index actions so documents can be pushed to Azure Search in batches. + // + // Also, throttle the pushes to Azure Search based on time so that we don't cause too much load. + var idsToIndex = new ConcurrentBag(); + var indexActionsToPush = new ConcurrentBag>(); + var timeSinceLastPush = new Stopwatch(); + var batchPushResults = new List(); + + while (idBag.TryTake(out var id)) + { + // FIRST, check if we have a full batch of package IDs to produce index actions for. + // + // If all of the IDs to index and the current ID were to need a document for each search filter and + // that number plus the current index actions to push would make the batch larger than the maximum + // batch size, produce index actions for the IDs that we have collected so far. + if (GetBatchSize(indexActionsToPush) + ((idsToIndex.Count + 1) * MaxDocumentsPerId) > _options.Value.AzureSearchBatchSize) + { + await GenerateIndexActionsAsync(idsToIndex, indexActionsToPush, changes); + } + + // SECOND, check if we have a full batch of index actions to push to Azure Search. + // + // If the current ID were to need a document for each search filter and the current batch size would + // make the batch larger than the maximum batch size, push the index actions we have so far. + if (GetBatchSize(indexActionsToPush) + MaxDocumentsPerId > _options.Value.AzureSearchBatchSize) + { + _logger.LogInformation( + "Starting to push a batch. There are {IdCount} unprocessed IDs left to index and push.", + idBag.Count); + batchPushResults.Add(await PushIndexActionsAsync(indexActionsToPush, timeSinceLastPush)); + } + + // THIRD, now that the two batching "buffers" have been flushed if necessary, add the current ID to the + // batch of IDs to produce index actions for. + idsToIndex.Add(id); + } + + // Process any leftover IDs that didn't make it into a full batch. + if (idsToIndex.Any()) + { + await GenerateIndexActionsAsync(idsToIndex, indexActionsToPush, changes); + } + + // Push any leftover index actions that didn't make it into a full batch. + if (indexActionsToPush.Any()) + { + batchPushResults.Add(await PushIndexActionsAsync(indexActionsToPush, timeSinceLastPush)); + } + + Guard.Assert(idsToIndex.IsEmpty, "There should be no more IDs to process."); + Guard.Assert(indexActionsToPush.IsEmpty, "There should be no more index actions to push."); + + return batchPushResults.Aggregate(new BatchPusherResult(), (a, b) => a.Merge(b)); + } + + /// + /// Generate index actions for each provided ID. This reads the version list per package ID so we want to + /// parallel this work by . + /// + private async Task GenerateIndexActionsAsync( + ConcurrentBag idsToIndex, + ConcurrentBag> indexActionsToPush, + SortedDictionary changes) + { + await ParallelAsync.Repeat( + async () => + { + while (idsToIndex.TryTake(out var id)) + { + var indexActions = await _indexActionBuilder.UpdateAsync( + id, + sf => _searchDocumentBuilder.UpdateDownloadCount(id, sf, changes[id])); + + if (indexActions.IsEmpty) + { + continue; + } + + Guard.Assert(indexActions.Hijack.Count == 0, "There should be no hijack index changes."); + + indexActionsToPush.Add(new IdAndValue(id, indexActions)); + } + }, + _options.Value.MaxConcurrentVersionListWriters); + } + + private async Task PushIndexActionsAsync( + ConcurrentBag> indexActionsToPush, + Stopwatch timeSinceLastPush) + { + var elapsed = timeSinceLastPush.Elapsed; + if (timeSinceLastPush.IsRunning && elapsed < _options.Value.MinPushPeriod) + { + var timeUntilNextPush = _options.Value.MinPushPeriod - elapsed; + _logger.LogInformation( + "Waiting for {Duration} before continuing.", + timeUntilNextPush); + await _systemTime.Delay(timeUntilNextPush); + } + + /// Create a new batch pusher just for this operation. Note that we don't use the internal queue of the + /// batch pusher for more than a single batch because we want to control exactly when batches are pushed to + /// Azure Search so that we can observe the + /// configuration property. + var batchPusher = _batchPusherFactory(); + + _logger.LogInformation( + "Pushing a batch of {IdCount} IDs and {DocumentCount} documents.", + indexActionsToPush.Count, + GetBatchSize(indexActionsToPush)); + + while (indexActionsToPush.TryTake(out var indexActions)) + { + batchPusher.EnqueueIndexActions(indexActions.Id, indexActions.Value); + } + + var finishResult = await batchPusher.TryFinishAsync(); + + // Restart the timer AFTER the push is completed to err on the side of caution. + timeSinceLastPush.Restart(); + + return finishResult; + } + + /// + /// This returns the number of documents that will be pushed to an Azure Search index in a + /// single batch. The caller is responsible that this number does not exceed + /// . If this job were to start pushing changes + /// to more than one index (more than the search index), then the number returned here should be the max of + /// the document counts per index. + /// + private int GetBatchSize(ConcurrentBag> indexActionsToPush) + { + return indexActionsToPush.Sum(x => x.Value.Search.Count); + } + + private void CleanDownloadData(DownloadData data) + { + var invalidIdCount = 0; + var invalidVersionCount = 0; + var nonNormalizedVersionCount = 0; + + foreach (var id in data.Keys.ToList()) + { + var isValidId = id.Length <= PackageIdValidator.MaxPackageIdLength && PackageIdValidator.IsValidPackageId(id); + if (!isValidId) + { + invalidIdCount++; + } + + foreach (var version in data[id].Keys.ToList()) + { + var isValidVersion = NuGetVersion.TryParse(version, out var parsedVersion); + if (!isValidVersion) + { + invalidVersionCount++; + } + + if (!isValidId || !isValidVersion) + { + // Clear the download count if the ID or version is invalid. + data.SetDownloadCount(id, version, 0); + continue; + } + + var normalizedVersion = parsedVersion.ToNormalizedString(); + var isNormalizedVersion = StringComparer.OrdinalIgnoreCase.Equals(version, normalizedVersion); + + if (!isNormalizedVersion) + { + nonNormalizedVersionCount++; + + // Use the normalized version string if the original was not normalized. + var downloads = data.GetDownloadCount(id, version); + data.SetDownloadCount(id, version, 0); + data.SetDownloadCount(id, normalizedVersion, downloads); + } + } + } + + _logger.LogInformation( + "There were {InvalidIdCount} invalid IDs, {InvalidVersionCount} invalid versions, and " + + "{NonNormalizedVersionCount} non-normalized IDs.", + invalidIdCount, + invalidVersionCount, + nonNormalizedVersionCount); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/UpdateOwnersCommand.cs b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/UpdateOwnersCommand.cs new file mode 100644 index 000000000..c02fc1cf8 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/UpdateOwnersCommand.cs @@ -0,0 +1,170 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch +{ + public class UpdateOwnersCommand : IAzureSearchCommand + { + private readonly IDatabaseAuxiliaryDataFetcher _databaseFetcher; + private readonly IOwnerDataClient _ownerDataClient; + private readonly IDataSetComparer _ownerSetComparer; + private readonly ISearchDocumentBuilder _searchDocumentBuilder; + private readonly ISearchIndexActionBuilder _searchIndexActionBuilder; + private readonly Func _batchPusherFactory; + private readonly IOptionsSnapshot _options; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + + public UpdateOwnersCommand( + IDatabaseAuxiliaryDataFetcher databaseFetcher, + IOwnerDataClient ownerDataClient, + IDataSetComparer ownerSetComparer, + ISearchDocumentBuilder searchDocumentBuilder, + ISearchIndexActionBuilder indexActionBuilder, + Func batchPusherFactory, + IOptionsSnapshot options, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _databaseFetcher = databaseFetcher ?? throw new ArgumentNullException(nameof(databaseFetcher)); + _ownerDataClient = ownerDataClient ?? throw new ArgumentNullException(nameof(ownerDataClient)); + _ownerSetComparer = ownerSetComparer ?? throw new ArgumentNullException(nameof(ownerSetComparer)); + _searchDocumentBuilder = searchDocumentBuilder ?? throw new ArgumentNullException(nameof(searchDocumentBuilder)); + _searchIndexActionBuilder = indexActionBuilder ?? throw new ArgumentNullException(nameof(indexActionBuilder)); + _batchPusherFactory = batchPusherFactory ?? throw new ArgumentNullException(nameof(batchPusherFactory)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (_options.Value.MaxConcurrentBatches <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(options), + $"The {nameof(AzureSearchJobConfiguration.MaxConcurrentBatches)} must be greater than zero."); + } + } + + public async Task ExecuteAsync() + { + var stopwatch = Stopwatch.StartNew(); + var outcome = JobOutcome.Failure; + try + { + _logger.LogInformation("Fetching old owner data from blob storage."); + var storageResult = await _ownerDataClient.ReadLatestIndexedAsync(); + + _logger.LogInformation("Fetching new owner data from the database."); + var databaseResult = await _databaseFetcher.GetPackageIdToOwnersAsync(); + + _logger.LogInformation("Detecting owner changes."); + var changes = _ownerSetComparer.CompareOwners(storageResult.Result, databaseResult); + var changesBag = new ConcurrentBag>(changes.Select(x => new IdAndValue(x.Key, x.Value))); + _logger.LogInformation("{Count} package IDs have owner changes.", changesBag.Count); + + if (!changes.Any()) + { + outcome = JobOutcome.NoOp; + return; + } + + _logger.LogInformation( + "Starting {Count} workers pushing owners changes to Azure Search.", + _options.Value.MaxConcurrentBatches); + await ParallelAsync.Repeat(() => WorkAndRetryAsync(changesBag), _options.Value.MaxConcurrentBatches); + _logger.LogInformation("All of the owner changes have been pushed to Azure Search."); + + // Persist in storage the list of all package IDs that have owner changes. This allows debugging and future + // analytics on frequency of ownership changes. + _logger.LogInformation("Uploading the package IDs that have owner changes to blob storage."); + await _ownerDataClient.UploadChangeHistoryAsync(changes.Keys.ToList()); + + _logger.LogInformation("Uploading the new owner data to blob storage."); + await _ownerDataClient.ReplaceLatestIndexedAsync(databaseResult, storageResult.AccessCondition); + outcome = JobOutcome.Success; + } + finally + { + stopwatch.Stop(); + _telemetryService.TrackUpdateOwnersCompleted(outcome, stopwatch.Elapsed); + } + } + + private async Task WorkAndRetryAsync(ConcurrentBag> changesBag) + { + await Task.Yield(); + + WorkResult result; + var attempt = 0; + do + { + attempt++; + result = await WorkAsync(changesBag); + changesBag = result.Attempted; + } + while (!result.Success && attempt < 3); + + result + .Results + .Where(x => !x.Success) + .Aggregate(new BatchPusherResult(), (a, b) => a.Merge(b)) + .EnsureSuccess(); + } + + private async Task WorkAsync(ConcurrentBag> changesBag) + { + var batchPusher = _batchPusherFactory(); + var attempted = new ConcurrentBag>(); + var results = new List(); + + while (changesBag.TryTake(out var changes)) + { + attempted.Add(changes); + + // Note that the owner list passed in can be empty (e.g. if the last owner was deleted or removed from + // the package registration). + var indexActions = await _searchIndexActionBuilder.UpdateAsync( + changes.Id, + searchFilters => _searchDocumentBuilder.UpdateOwners(changes.Id, searchFilters, changes.Value)); + + // If no index actions are returned, this means that there are no listed packages or no + // packages at all. + if (indexActions.IsEmpty) + { + continue; + } + + batchPusher.EnqueueIndexActions(changes.Id, indexActions); + + results.Add(await batchPusher.TryPushFullBatchesAsync()); + } + + results.Add(await batchPusher.TryFinishAsync()); + return new WorkResult(attempted, results); + } + + private class WorkResult + { + public WorkResult(ConcurrentBag> attempted, List results) + { + Attempted = attempted; + Results = results; + } + + public bool Success => Results.All(x => x.Success); + public ConcurrentBag> Attempted { get; } + public List Results { get; } + } + } +} + diff --git a/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/UpdateVerifiedPackagesCommand.cs b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/UpdateVerifiedPackagesCommand.cs new file mode 100644 index 000000000..8d0402b26 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Auxiliary2AzureSearch/UpdateVerifiedPackagesCommand.cs @@ -0,0 +1,76 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch +{ + public class UpdateVerifiedPackagesCommand : IAzureSearchCommand + { + private readonly IDatabaseAuxiliaryDataFetcher _databaseFetcher; + private readonly IVerifiedPackagesDataClient _verifiedPackagesDataClient; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + private readonly StringCache _stringCache; + + public UpdateVerifiedPackagesCommand( + IDatabaseAuxiliaryDataFetcher databaseFetcher, + IVerifiedPackagesDataClient verifiedPackagesDataClient, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _databaseFetcher = databaseFetcher ?? throw new ArgumentNullException(nameof(databaseFetcher)); + _verifiedPackagesDataClient = verifiedPackagesDataClient ?? throw new ArgumentNullException(nameof(verifiedPackagesDataClient)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _stringCache = new StringCache(); + } + + public async Task ExecuteAsync() + { + var stopwatch = Stopwatch.StartNew(); + var outcome = JobOutcome.Failure; + try + { + outcome = await UpdateVerifiedPackagesAsync() ? JobOutcome.Success : JobOutcome.NoOp; + } + finally + { + stopwatch.Stop(); + _telemetryService.TrackUpdateVerifiedPackagesCompleted(outcome, stopwatch.Elapsed); + } + } + + private async Task UpdateVerifiedPackagesAsync() + { + // The "old" data in this case is the latest file that was copied to the region's storage container by this + // job (or initialized by Db2AzureSearch). + var oldResult = await _verifiedPackagesDataClient.ReadLatestAsync( + AccessConditionWrapper.GenerateEmptyCondition(), + _stringCache); + + // The "new" data in this case is from the database. + var newData = await _databaseFetcher.GetVerifiedPackagesAsync(); + + var changes = new HashSet(oldResult.Data, oldResult.Data.Comparer); + changes.SymmetricExceptWith(newData); + _logger.LogInformation("{Count} package IDs have verified status changes.", changes.Count); + + if (changes.Count == 0) + { + return false; + } + else + { + await _verifiedPackagesDataClient.ReplaceLatestAsync(newData, oldResult.Metadata.GetIfMatchCondition()); + return true; + } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryDataStorageConfiguration.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryDataStorageConfiguration.cs new file mode 100644 index 000000000..8abe9ca42 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryDataStorageConfiguration.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class AuxiliaryDataStorageConfiguration : IAuxiliaryDataStorageConfiguration + { + public string AuxiliaryDataStorageConnectionString { get; set; } + public string AuxiliaryDataStorageContainer { get; set; } + public string AuxiliaryDataStorageDownloadsPath { get; set; } + public string AuxiliaryDataStorageExcludedPackagesPath { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryFileClient.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryFileClient.cs new file mode 100644 index 000000000..e68014de6 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryFileClient.cs @@ -0,0 +1,102 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Newtonsoft.Json; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class AuxiliaryFileClient : IAuxiliaryFileClient + { + private readonly ICloudBlobClient _cloudBlobClient; + private readonly IOptionsSnapshot _options; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + private readonly Lazy _lazyContainer; + + public AuxiliaryFileClient( + ICloudBlobClient cloudBlobClient, + IOptionsSnapshot options, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _cloudBlobClient = cloudBlobClient ?? throw new ArgumentNullException(nameof(cloudBlobClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + _lazyContainer = new Lazy( + () => _cloudBlobClient.GetContainerReference(_options.Value.AuxiliaryDataStorageContainer)); + } + + private ICloudBlobContainer Container => _lazyContainer.Value; + + public async Task LoadDownloadDataAsync() + { + return await LoadAuxiliaryFileAsync( + _options.Value.AuxiliaryDataStorageDownloadsPath, + loadData: reader => + { + var downloadData = new DownloadData(); + DownloadsV1Reader.Load(reader, downloadData.SetDownloadCount); + return downloadData; + }); + } + + public async Task> LoadExcludedPackagesAsync() + { + return await LoadAuxiliaryFileAsync( + _options.Value.AuxiliaryDataStorageExcludedPackagesPath, + reader => JsonStringArrayFileParser.Load(reader, _logger)); + } + + private async Task LoadAuxiliaryFileAsync( + string blobName, + Func loadData) where T : class + { + // Retry on HTTP 412 because it's possible CloudBlockBlob.OpenReadyAsync throws a 412 while reading the + // resulting stream. This is because the WindowsAzure.Storage SDK implements the streaming read with a + // series of range requests with If-Match, presumably to keep memory consumption at a minimum. During these + // reads, the file can be modified which will cause an HTTP 412 Precondition Failed. + var data = default(T); + await Retry.IncrementalAsync( + async () => + { + _logger.LogInformation( + "Attempted to load blob {BlobName} as {TypeName}.", + blobName, + typeof(T).FullName); + + var stopwatch = Stopwatch.StartNew(); + var blob = Container.GetBlobReference(blobName); + using (var stream = await blob.OpenReadAsync(AccessCondition.GenerateEmptyCondition())) + using (var textReader = new StreamReader(stream)) + using (var jsonReader = new JsonTextReader(textReader)) + { + data = loadData(jsonReader); + stopwatch.Stop(); + + _telemetryService.TrackAuxiliaryFileDownloaded(blobName, stopwatch.Elapsed); + _logger.LogInformation( + "Loaded blob {BlobName}. Took {Duration}.", + blobName, + stopwatch.Elapsed); + }; + }, + ex => ex is StorageException se && se.IsPreconditionFailedException(), + maxRetries: 5, + initialWaitInterval: TimeSpan.Zero, + waitIncrement: TimeSpan.FromSeconds(10)); + return data; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryFileMetadata.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryFileMetadata.cs new file mode 100644 index 000000000..0f294938e --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryFileMetadata.cs @@ -0,0 +1,35 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class AuxiliaryFileMetadata + { + [JsonConstructor] + public AuxiliaryFileMetadata( + DateTimeOffset lastModified, + TimeSpan loadDuration, + long fileSize, + string etag) + { + LastModified = lastModified; + LoadDuration = loadDuration; + FileSize = fileSize; + ETag = etag ?? throw new ArgumentNullException(nameof(etag)); + } + + public DateTimeOffset LastModified { get; } + public TimeSpan LoadDuration { get; } + public long FileSize { get; } + public string ETag { get; } + + public IAccessCondition GetIfMatchCondition() + { + return AccessConditionWrapper.GenerateIfMatchCondition(ETag); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryFileResult.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryFileResult.cs new file mode 100644 index 000000000..c203b8fe9 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/AuxiliaryFileResult.cs @@ -0,0 +1,54 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class AuxiliaryFileResult where T : class + { + public AuxiliaryFileResult( + bool modified, + T data, + AuxiliaryFileMetadata metadata) + { + Modified = modified; + if (modified) + { + Data = data ?? throw new ArgumentNullException(nameof(data)); + Metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); + } + else + { + if (data != null) + { + throw new ArgumentException("The fetched data must be null if it was not modified.", nameof(data)); + } + + if (metadata != null) + { + throw new ArgumentException("The file metadata must be null if it was not modified.", nameof(data)); + } + } + } + + /// + /// Whether or not the data has been modified since the last time this result was fetched. This can be set to + /// false by an auxiliary file client by reading an endpoint with an etag, i.e. with the If-Match: HTTP + /// request header. + /// + public bool Modified { get; } + + /// + /// The data in the auxiliary file. This will be non-null if is true and null if it is + /// false. + /// + public T Data { get; } + + /// + /// The metadata about the auxiliary file for no-op and diagnostics purposes. This type has the etag which can + /// be used to only download the data if it has changed. + /// + public AuxiliaryFileMetadata Metadata { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadByVersionData.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadByVersionData.cs new file mode 100644 index 000000000..3fed4b052 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadByVersionData.cs @@ -0,0 +1,64 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections; +using System.Collections.Generic; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class DownloadByVersionData : IReadOnlyDictionary + { + private readonly Dictionary _versions + = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public long Total { get; private set; } + + public long GetDownloadCount(string version) + { + if (!_versions.TryGetValue(version, out var downloads)) + { + return 0; + } + + return downloads; + } + + public void SetDownloadCount(string version, long downloads) + { + if (downloads < 0) + { + throw new ArgumentOutOfRangeException(nameof(downloads), "The download count must not be negative."); + } + + if (_versions.TryGetValue(version, out var existingDownloads)) + { + // Remove the previous version so that the latest case is retained. Versions are case insensitive but + // we should try to respect the latest intent. + _versions.Remove(version); + } + else + { + existingDownloads = 0; + } + + Total += downloads - existingDownloads; + + // Only store the download count if the value is not zero. + if (downloads != 0) + { + _versions.Add(version, downloads); + } + } + + public IEnumerable Keys => _versions.Keys; + public IEnumerable Values => _versions.Values; + public int Count => _versions.Count; + public long this[string key] => _versions[key]; + public IEnumerator> GetEnumerator() => _versions.GetEnumerator(); + public bool TryGetValue(string key, out long value) => _versions.TryGetValue(key, out value); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + public bool ContainsKey(string key) => _versions.ContainsKey(key); + } +} + diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadData.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadData.cs new file mode 100644 index 000000000..4a4e61660 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadData.cs @@ -0,0 +1,72 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections; +using System.Collections.Generic; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class DownloadData : IReadOnlyDictionary + { + private readonly Dictionary _ids + = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public long GetDownloadCount(string id) + { + if (!_ids.TryGetValue(id, out var versionData)) + { + return 0; + } + + return versionData.Total; + } + + public long GetDownloadCount(string id, string version) + { + if (!_ids.TryGetValue(id, out var versionData)) + { + return 0; + } + + return versionData.GetDownloadCount(version); + } + + public void SetDownloadCount(string id, string version, long downloads) + { + if (downloads < 0) + { + throw new ArgumentOutOfRangeException(nameof(downloads), "The download count must not be negative."); + } + + if (_ids.TryGetValue(id, out var versions)) + { + // Remove the previous version so that the latest case is retained. IDs are case insensitive but we + // should try to respect the latest intent. + _ids.Remove(id); + } + else + { + versions = new DownloadByVersionData(); + } + + versions.SetDownloadCount(version, downloads); + + // Only store the download count if the value is not zero. + if (versions.Total != 0) + { + _ids.Add(id, versions); + } + } + + public IEnumerable Keys => _ids.Keys; + public IEnumerable Values => _ids.Values; + public int Count => _ids.Count; + public DownloadByVersionData this[string key] => _ids[key]; + public IEnumerator> GetEnumerator() => _ids.GetEnumerator(); + public bool TryGetValue(string key, out DownloadByVersionData value) => _ids.TryGetValue(key, out value); + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + public bool ContainsKey(string key) => _ids.ContainsKey(key); + } +} + diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadDataClient.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadDataClient.cs new file mode 100644 index 000000000..bd8492213 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadDataClient.cs @@ -0,0 +1,165 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.IO; +using System.Net; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Newtonsoft.Json; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class DownloadDataClient : IDownloadDataClient + { + private static readonly JsonSerializer Serializer = new JsonSerializer(); + + private readonly ICloudBlobClient _cloudBlobClient; + private readonly IOptionsSnapshot _options; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + private readonly Lazy _lazyContainer; + + public DownloadDataClient( + ICloudBlobClient cloudBlobClient, + IOptionsSnapshot options, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _cloudBlobClient = cloudBlobClient ?? throw new ArgumentNullException(nameof(cloudBlobClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + _lazyContainer = new Lazy( + () => _cloudBlobClient.GetContainerReference(_options.Value.StorageContainer)); + } + + private ICloudBlobContainer Container => _lazyContainer.Value; + + public async Task> ReadLatestIndexedAsync( + IAccessCondition accessCondition, + StringCache stringCache) + { + var stopwatch = Stopwatch.StartNew(); + var blobName = GetLatestIndexedBlobName(); + var blobReference = Container.GetBlobReference(blobName); + + _logger.LogInformation("Reading the latest indexed downloads from {BlobName}.", blobName); + + bool modified; + var downloads = new DownloadData(); + AuxiliaryFileMetadata metadata; + try + { + using (var stream = await blobReference.OpenReadAsync(accessCondition)) + { + ReadStream( + stream, + (id, version, downloadCount) => + { + id = stringCache.Dedupe(id); + version = stringCache.Dedupe(version); + downloads.SetDownloadCount(id, version, downloadCount); + }); + modified = true; + metadata = new AuxiliaryFileMetadata( + lastModified: new DateTimeOffset(blobReference.LastModifiedUtc, TimeSpan.Zero), + loadDuration: stopwatch.Elapsed, + fileSize: blobReference.Properties.Length, + etag: blobReference.ETag); + } + } + catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotModified) + { + _logger.LogInformation("The blob {BlobName} has not changed.", blobName); + modified = false; + downloads = null; + metadata = null; + } + + stopwatch.Stop(); + _telemetryService.TrackReadLatestIndexedDownloads(downloads?.Count, modified, stopwatch.Elapsed); + + return new AuxiliaryFileResult( + modified, + downloads, + metadata); + } + + public async Task ReplaceLatestIndexedAsync( + DownloadData newData, + IAccessCondition accessCondition) + { + using (_telemetryService.TrackReplaceLatestIndexedDownloads(newData.Count)) + { + var blobName = GetLatestIndexedBlobName(); + _logger.LogInformation("Replacing the latest indexed downloads from {BlobName}.", blobName); + + var blobReference = Container.GetBlobReference(blobName); + + using (var stream = await blobReference.OpenWriteAsync(accessCondition)) + using (var streamWriter = new StreamWriter(stream)) + using (var jsonTextWriter = new JsonTextWriter(streamWriter)) + { + blobReference.Properties.ContentType = "application/json"; + Serializer.Serialize(jsonTextWriter, newData); + } + } + } + + private static void ReadStream( + Stream stream, + Action addVersion) + { + using (var textReader = new StreamReader(stream)) + using (var jsonReader = new JsonTextReader(textReader)) + { + Guard.Assert(jsonReader.Read(), "The blob should be readable."); + Guard.Assert(jsonReader.TokenType == JsonToken.StartObject, "The first token should be the start of an object."); + Guard.Assert(jsonReader.Read(), "There should be a second token."); + + while (jsonReader.TokenType == JsonToken.PropertyName) + { + // We assume the package ID has valid characters. + var id = (string)jsonReader.Value; + + Guard.Assert(jsonReader.Read(), "There should be a token after the package ID."); + Guard.Assert(jsonReader.TokenType == JsonToken.StartObject, "The token after the package ID should be the start of an object."); + Guard.Assert(jsonReader.Read(), "There should be a token after the start of the ID object."); + + while (jsonReader.TokenType == JsonToken.PropertyName) + { + // We assume the package version is already normalized. + var version = (string)jsonReader.Value; + + Guard.Assert(jsonReader.Read(), "There should be a token after the package version."); + Guard.Assert(jsonReader.TokenType == JsonToken.Integer, "The token after the package version should be an integer."); + + var downloads = (long)jsonReader.Value; + + Guard.Assert(jsonReader.Read(), "There should be a token after the download count."); + + addVersion(id, version, downloads); + } + + Guard.Assert(jsonReader.TokenType == JsonToken.EndObject, "The token after the package versions should be the end of an object."); + Guard.Assert(jsonReader.Read(), "There should be a token after the package ID object."); + } + + Guard.Assert(jsonReader.TokenType == JsonToken.EndObject, "The last token should be the end of an object."); + Guard.Assert(!jsonReader.Read(), "There should be no token after the end of the object."); + } + } + + private string GetLatestIndexedBlobName() + { + return $"{_options.Value.NormalizeStoragePath()}downloads/downloads.v2.json"; + } + } +} + diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadsV1Reader.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadsV1Reader.cs new file mode 100644 index 000000000..7ece6bf3f --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/DownloadsV1Reader.cs @@ -0,0 +1,49 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public static class DownloadsV1Reader + { + public static void Load(JsonReader jsonReader, Action addCount) + { + // The data in downloads.v1.json will be an array of Package records - which has Id, Array of Versions and download count. + // Sample.json : [["AutofacContrib.NSubstitute",["2.4.3.700",406],["2.5.0",137]],["Assman.Core",["2.0.7",138]].... + jsonReader.Read(); + + while (jsonReader.Read()) + { + if (jsonReader.TokenType == JsonToken.StartArray) + { + JToken record = JToken.ReadFrom(jsonReader); + + // The second entry in each record should be an array of versions, if not move on to next entry. + // This is a check to safe guard against invalid entries. + if (record.Count() == 2 && record[1].Type != JTokenType.Array) + { + continue; + } + + var id = record[0].ToString(); + + foreach (JToken token in record) + { + if (token != null && token.Count() == 2) + { + var version = token[0].ToString(); + + var count = token[1].ToObject(); + + addCount.Invoke(id, version, count); + } + } + } + } + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IAuxiliaryDataStorageConfiguration.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IAuxiliaryDataStorageConfiguration.cs new file mode 100644 index 000000000..8a44c4dfa --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IAuxiliaryDataStorageConfiguration.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public interface IAuxiliaryDataStorageConfiguration + { + string AuxiliaryDataStorageConnectionString { get; } + string AuxiliaryDataStorageContainer { get; } + string AuxiliaryDataStorageDownloadsPath { get; } + string AuxiliaryDataStorageExcludedPackagesPath { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IAuxiliaryFileClient.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IAuxiliaryFileClient.cs new file mode 100644 index 000000000..b284df2b7 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IAuxiliaryFileClient.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public interface IAuxiliaryFileClient + { + Task LoadDownloadDataAsync(); + Task> LoadExcludedPackagesAsync(); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IDownloadDataClient.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IDownloadDataClient.cs new file mode 100644 index 000000000..194b88b2b --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IDownloadDataClient.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public interface IDownloadDataClient + { + Task> ReadLatestIndexedAsync(IAccessCondition accessCondition, StringCache stringCache); + Task ReplaceLatestIndexedAsync(DownloadData newData, IAccessCondition accessCondition); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IOwnerDataClient.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IOwnerDataClient.cs new file mode 100644 index 000000000..a0d8d635b --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IOwnerDataClient.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + /// + /// The purpose of this interface is allow reading and writing owner information from storage. The Catalog2Owners + /// job does a comparison of latest owner data from the database with a snapshot of information stored in Azure + /// Blob Storage. This interface handles the reading and writing of that snapshot from storage. + /// + public interface IOwnerDataClient + { + /// + /// Read all of the latest indexed owners from storage. Also, return the current etag to allow optimistic + /// concurrency checks on the writing of the file. The returned dictionary has a key which is the package ID + /// and a value which is the owners of that package ID. The dictionary and the sets are case-insensitive. + /// + Task>>> ReadLatestIndexedAsync(); + + /// + /// Replace the existing latest indexed owners file (i.e. "owners.v2.json" file). + /// + /// The new data to be serialized into storage. + /// The access condition (i.e. etag) to use during the upload. + Task ReplaceLatestIndexedAsync( + SortedDictionary> newData, + IAccessCondition accessCondition); + + /// + /// Write a list of owners to storage. The file name that will be used in storage will be a timestamp so + /// subsequent calls should not conflict. + /// + /// A non-empty list of package IDs that had owner changes. + Task UploadChangeHistoryAsync(IReadOnlyList packageIds); + } +} + diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IPopularityTransferDataClient.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IPopularityTransferDataClient.cs new file mode 100644 index 000000000..33ed9fb13 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IPopularityTransferDataClient.cs @@ -0,0 +1,35 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + /// + /// The purpose of this interface is allow reading and writing popularity transfer information from storage. + /// The Auxiliary2AzureSearch job does a comparison of latest popularity transfer data from the database with + /// a snapshot of information stored in Azure Blob Storage. This interface handles the reading and writing of + /// that snapshot from storage. + /// + public interface IPopularityTransferDataClient + { + /// + /// Read all of the latest indexed popularity transfers from storage. Also, return the current etag to allow + /// optimistic concurrency checks on the writing of the file. + /// + Task> ReadLatestIndexedAsync( + IAccessCondition accessCondition, + StringCache stringCache); + + /// + /// Replace the existing latest indexed popularity transfers file (i.e. "popularityTransfers.v1.json" file). + /// + /// The new data to be serialized into storage. + /// The access condition (i.e. etag) to use during the upload. + Task ReplaceLatestIndexedAsync( + PopularityTransferData newData, + IAccessCondition accessCondition); + } +} + diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IVerifiedPackagesDataClient.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IVerifiedPackagesDataClient.cs new file mode 100644 index 000000000..acd1733c2 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/IVerifiedPackagesDataClient.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public interface IVerifiedPackagesDataClient + { + Task>> ReadLatestAsync(IAccessCondition accessCondition, StringCache stringCache); + Task ReplaceLatestAsync(HashSet newData, IAccessCondition accessCondition); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/JsonStringArrayFileParser.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/JsonStringArrayFileParser.cs new file mode 100644 index 000000000..46f7dcbbe --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/JsonStringArrayFileParser.cs @@ -0,0 +1,72 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public static class JsonStringArrayFileParser + { + /// + /// Load the auxiliary data in simple json string array format. + /// + /// The name of the file that contains the auxiliary data + /// The loader that should be used to fetch the file's content + /// The logger + /// A case-insensitive set of all the strings in the json array + public static HashSet Load(JsonReader reader, ILogger logger) + { + try + { + return Parse(reader); + } + catch (Exception ex) + { + logger.LogError(ex, "Unable to load JSON string array."); + + throw; + } + } + + /// + /// Parse the string from the input. + /// + /// The reader whose content should be parsed + /// A case-insensitive set of all the verified packages + public static HashSet Parse(JsonReader reader) + { + // The file should contain an array of strings, such as: ["Package1","Package2", ...] + reader.Read(); + ThrowIfNotExpectedToken(reader, JsonToken.StartArray); + + // Read all of the strings from the JSON array. + var result = new HashSet(StringComparer.OrdinalIgnoreCase); + var stringValue = reader.ReadAsString(); + + while (stringValue != null) + { + // Package IDs strings are likely to be duplicates from previous reloads. We'll reuse the + // interned strings so that duplicated strings can be garbage collected right away. + result.Add(String.Intern(stringValue)); + + stringValue = reader.ReadAsString(); + } + + ThrowIfNotExpectedToken(reader, JsonToken.EndArray); + + return result; + } + + private static void ThrowIfNotExpectedToken(JsonReader reader, JsonToken expected) + { + if (reader.TokenType != expected) + { + throw new InvalidDataException($"Malformed simple json string array auxiliary file - expected '{JsonToken.StartArray}', actual: '{reader.TokenType}'"); + } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/OwnerDataClient.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/OwnerDataClient.cs new file mode 100644 index 000000000..adb6b7793 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/OwnerDataClient.cs @@ -0,0 +1,164 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Net; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Newtonsoft.Json; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class OwnerDataClient : IOwnerDataClient + { + private static readonly JsonSerializer Serializer = new JsonSerializer(); + + private readonly ICloudBlobClient _cloudBlobClient; + private readonly IOptionsSnapshot _options; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + private readonly Lazy _lazyContainer; + + public OwnerDataClient( + ICloudBlobClient cloudBlobClient, + IOptionsSnapshot options, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _cloudBlobClient = cloudBlobClient ?? throw new ArgumentNullException(nameof(cloudBlobClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + _lazyContainer = new Lazy( + () => _cloudBlobClient.GetContainerReference(_options.Value.StorageContainer)); + } + + private ICloudBlobContainer Container => _lazyContainer.Value; + + public async Task>>> ReadLatestIndexedAsync() + { + var stopwatch = Stopwatch.StartNew(); + var blobName = GetLatestIndexedBlobName(); + var blobReference = Container.GetBlobReference(blobName); + + _logger.LogInformation("Reading the latest indexed owners from {BlobName}.", blobName); + + var builder = new PackageIdToOwnersBuilder(_logger); + IAccessCondition accessCondition; + try + { + using (var stream = await blobReference.OpenReadAsync(AccessCondition.GenerateEmptyCondition())) + { + accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blobReference.ETag); + ReadStream(stream, builder.Add); + } + } + catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound) + { + accessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition(); + _logger.LogInformation("The blob {BlobName} does not exist.", blobName); + } + + var output = new ResultAndAccessCondition>>( + builder.GetResult(), + accessCondition); + + stopwatch.Stop(); + _telemetryService.TrackReadLatestIndexedOwners(output.Result.Count, stopwatch.Elapsed); + + return output; + } + + public async Task UploadChangeHistoryAsync(IReadOnlyList packageIds) + { + if (packageIds.Count == 0) + { + throw new ArgumentException("The list of package IDs must have at least one element.", nameof(packageIds)); + } + + using (_telemetryService.TrackUploadOwnerChangeHistory(packageIds.Count)) + { + var timestamp = DateTimeOffset.UtcNow.ToString("yyyy-MM-dd-HH-mm-ss-FFFFFFF"); + var blobName = $"{_options.Value.NormalizeStoragePath()}owners/changes/{timestamp}.json"; + _logger.LogInformation("Uploading owner changes to {BlobName}.", blobName); + + var blobReference = Container.GetBlobReference(blobName); + + using (var stream = await blobReference.OpenWriteAsync(AccessCondition.GenerateIfNotExistsCondition())) + using (var streamWriter = new StreamWriter(stream)) + using (var jsonTextWriter = new JsonTextWriter(streamWriter)) + { + blobReference.Properties.ContentType = "application/json"; + Serializer.Serialize(jsonTextWriter, packageIds); + } + } + } + + public async Task ReplaceLatestIndexedAsync( + SortedDictionary> newData, + IAccessCondition accessCondition) + { + using (_telemetryService.TrackReplaceLatestIndexedOwners(newData.Count)) + { + var blobName = GetLatestIndexedBlobName(); + _logger.LogInformation("Replacing the latest indexed owners from {BlobName}.", blobName); + + var mappedAccessCondition = new AccessCondition + { + IfNoneMatchETag = accessCondition.IfNoneMatchETag, + IfMatchETag = accessCondition.IfMatchETag, + }; + + var blobReference = Container.GetBlobReference(blobName); + + using (var stream = await blobReference.OpenWriteAsync(mappedAccessCondition)) + using (var streamWriter = new StreamWriter(stream)) + using (var jsonTextWriter = new JsonTextWriter(streamWriter)) + { + blobReference.Properties.ContentType = "application/json"; + Serializer.Serialize(jsonTextWriter, newData); + } + } + } + + private static void ReadStream(Stream stream, Action> add) + { + using (var textReader = new StreamReader(stream)) + using (var jsonReader = new JsonTextReader(textReader)) + { + Guard.Assert(jsonReader.Read(), "The blob should be readable."); + Guard.Assert(jsonReader.TokenType == JsonToken.StartObject, "The first token should be the start of an object."); + Guard.Assert(jsonReader.Read(), "There should be a second token."); + while (jsonReader.TokenType == JsonToken.PropertyName) + { + var id = (string)jsonReader.Value; + + Guard.Assert(jsonReader.Read(), "There should be a token after the property name."); + Guard.Assert(jsonReader.TokenType == JsonToken.StartArray, "The token after the property name should be the start of an object."); + + var owners = Serializer.Deserialize>(jsonReader); + add(id, owners); + + Guard.Assert(jsonReader.TokenType == JsonToken.EndArray, "The token after reading the array should be the end of an array."); + Guard.Assert(jsonReader.Read(), "There should be a token after the end of the array."); + } + + Guard.Assert(jsonReader.TokenType == JsonToken.EndObject, "The last token should be the end of an object."); + Guard.Assert(!jsonReader.Read(), "There should be no token after the end of the object."); + } + } + + private string GetLatestIndexedBlobName() + { + return $"{_options.Value.NormalizeStoragePath()}owners/owners.v2.json"; + } + } +} + diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/PopularityTransferData.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/PopularityTransferData.cs new file mode 100644 index 000000000..fa710ced4 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/PopularityTransferData.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections; +using System.Collections.Generic; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + /// + /// Maps packages that transfer their popularity away to the + /// set of packages receiving the popularity. + /// + public class PopularityTransferData : IReadOnlyDictionary> + { + private readonly SortedDictionary> _transfers = + new SortedDictionary>(StringComparer.OrdinalIgnoreCase); + + public void AddTransfer(string fromId, string toId) + { + if (!_transfers.TryGetValue(fromId, out var toIds)) + { + toIds = new SortedSet(StringComparer.OrdinalIgnoreCase); + _transfers.Add(fromId, toIds); + } + + toIds.Add(toId); + } + + public SortedSet this[string key] => _transfers[key]; + public IEnumerable Keys => _transfers.Keys; + public IEnumerable> Values => _transfers.Values; + public int Count => _transfers.Count; + public bool ContainsKey(string key) => _transfers.ContainsKey(key); + public IEnumerator>> GetEnumerator() => _transfers.GetEnumerator(); + public bool TryGetValue(string key, out SortedSet value) => _transfers.TryGetValue(key, out value); + IEnumerator IEnumerable.GetEnumerator() => _transfers.GetEnumerator(); + } +} diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/PopularityTransferDataClient.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/PopularityTransferDataClient.cs new file mode 100644 index 000000000..641a3e5d9 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/PopularityTransferDataClient.cs @@ -0,0 +1,153 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.IO; +using System.Net; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Newtonsoft.Json; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class PopularityTransferDataClient : IPopularityTransferDataClient + { + private static readonly JsonSerializer Serializer = new JsonSerializer(); + + private readonly ICloudBlobClient _cloudBlobClient; + private readonly IOptionsSnapshot _options; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + private readonly Lazy _lazyContainer; + + public PopularityTransferDataClient( + ICloudBlobClient cloudBlobClient, + IOptionsSnapshot options, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _cloudBlobClient = cloudBlobClient ?? throw new ArgumentNullException(nameof(cloudBlobClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + _lazyContainer = new Lazy( + () => _cloudBlobClient.GetContainerReference(_options.Value.StorageContainer)); + } + + private ICloudBlobContainer Container => _lazyContainer.Value; + + public async Task> ReadLatestIndexedAsync( + IAccessCondition accessCondition, + StringCache stringCache) + { + var stopwatch = Stopwatch.StartNew(); + var blobName = GetLatestIndexedBlobName(); + var blobReference = Container.GetBlobReference(blobName); + + _logger.LogInformation("Reading the latest indexed popularity transfers from {BlobName}.", blobName); + + bool modified; + var data = new PopularityTransferData(); + AuxiliaryFileMetadata metadata; + try + { + using (var stream = await blobReference.OpenReadAsync(accessCondition)) + { + ReadStream(stream, (from, to) => data.AddTransfer(stringCache.Dedupe(from), stringCache.Dedupe(to))); + modified = true; + metadata = new AuxiliaryFileMetadata( + lastModified: new DateTimeOffset(blobReference.LastModifiedUtc, TimeSpan.Zero), + loadDuration: stopwatch.Elapsed, + fileSize: blobReference.Properties.Length, + etag: blobReference.ETag); + } + } + catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotModified) + { + _logger.LogInformation("The blob {BlobName} has not changed.", blobName); + modified = false; + data = null; + metadata = null; + } + + stopwatch.Stop(); + _telemetryService.TrackReadLatestIndexedPopularityTransfers(data?.Count, modified, stopwatch.Elapsed); + + return new AuxiliaryFileResult( + modified, + data, + metadata); + } + + public async Task ReplaceLatestIndexedAsync( + PopularityTransferData newData, + IAccessCondition accessCondition) + { + using (_telemetryService.TrackReplaceLatestIndexedPopularityTransfers(newData.Count)) + { + var blobName = GetLatestIndexedBlobName(); + _logger.LogInformation("Replacing the latest indexed popularity transfers from {BlobName}.", blobName); + + var mappedAccessCondition = new AccessCondition + { + IfNoneMatchETag = accessCondition.IfNoneMatchETag, + IfMatchETag = accessCondition.IfMatchETag, + }; + + var blobReference = Container.GetBlobReference(blobName); + + using (var stream = await blobReference.OpenWriteAsync(mappedAccessCondition)) + using (var streamWriter = new StreamWriter(stream)) + using (var jsonTextWriter = new JsonTextWriter(streamWriter)) + { + blobReference.Properties.ContentType = "application/json"; + Serializer.Serialize(jsonTextWriter, newData); + } + } + } + + private static void ReadStream(Stream stream, Action add) + { + using (var textReader = new StreamReader(stream)) + using (var jsonReader = new JsonTextReader(textReader)) + { + Guard.Assert(jsonReader.Read(), "The blob should be readable."); + Guard.Assert(jsonReader.TokenType == JsonToken.StartObject, "The first token should be the start of an object."); + Guard.Assert(jsonReader.Read(), "There should be a second token."); + + while (jsonReader.TokenType == JsonToken.PropertyName) + { + var fromId = (string)jsonReader.Value; + + Guard.Assert(jsonReader.Read(), "There should be a token after the property name."); + Guard.Assert(jsonReader.TokenType == JsonToken.StartArray, "The token after the property name should be the start of an array."); + Guard.Assert(jsonReader.Read(), "There should be a token after the start of the transfer array."); + + while (jsonReader.TokenType == JsonToken.String) + { + add(fromId, (string)jsonReader.Value); + + Guard.Assert(jsonReader.Read(), "There should be a token after the 'to' package ID."); + } + + Guard.Assert(jsonReader.TokenType == JsonToken.EndArray, "The token after reading the array should be the end of an array."); + Guard.Assert(jsonReader.Read(), "There should be a token after the end of the array."); + } + + Guard.Assert(jsonReader.TokenType == JsonToken.EndObject, "The last token should be the end of an object."); + Guard.Assert(!jsonReader.Read(), "There should be no token after the end of the object."); + } + } + + private string GetLatestIndexedBlobName() + { + return $"{_options.Value.NormalizeStoragePath()}popularity-transfers/popularity-transfers.v1.json"; + } + } +} + diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/SimpleCloudBlobExtensions.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/SimpleCloudBlobExtensions.cs new file mode 100644 index 000000000..84b416bc8 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/SimpleCloudBlobExtensions.cs @@ -0,0 +1,32 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.IO; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + internal static class SimpleCloudBlobExtensions + { + public static async Task OpenReadAsync(this ISimpleCloudBlob blob, IAccessCondition accessCondition) + { + return await blob.OpenReadAsync(MapAccessCondition(accessCondition)); + } + + public static async Task OpenWriteAsync(this ISimpleCloudBlob blob, IAccessCondition accessCondition) + { + return await blob.OpenWriteAsync(MapAccessCondition(accessCondition)); + } + + private static AccessCondition MapAccessCondition(IAccessCondition accessCondition) + { + return new AccessCondition + { + IfNoneMatchETag = accessCondition.IfNoneMatchETag, + IfMatchETag = accessCondition.IfMatchETag, + }; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/StringCache.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/StringCache.cs new file mode 100644 index 000000000..b67971237 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/StringCache.cs @@ -0,0 +1,78 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Concurrent; +using System.Threading; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class StringCache + { + /// + /// Maintain a lookup of strings for de-duping. We maintain the original case for de-duping purposes by using + /// the default string comparer. As of July of 2019 in PROD, maintaining original case of version + /// string adds less than 0.3% extra strings. De-duping version strings in a case-sensitive manner removes + /// 87.0% of the string allocations. Intuitively this means most people use the same case of a given version + /// string and a lot of people use the same versions strings (common ones are 1.0.0, 1.0.1, 1.0.2, 1.1.0, etc). + /// + private readonly ConcurrentDictionary _values = new ConcurrentDictionary(); + + /// + /// Keep track of the number of requests for a string. This is the number of times + /// has been called. + /// + private int _requestCount = 0; + + /// + /// Keep track of the number of string de-duped, i.e. "cache hits". + /// + private int _hitCount = 0; + + /// + /// Keep track of the number of characters in the cache. + /// + private long _charCount = 0; + + public int StringCount => _values.Count; + public int RequestCount => _requestCount; + public int HitCount => _hitCount; + public long CharCount => _charCount; + + public string Dedupe(string value) + { + Interlocked.Increment(ref _requestCount); + + if (value == null) + { + return null; + } + + // Inspired by: + // https://devblogs.microsoft.com/pfxteam/building-a-custom-getoradd-method-for-concurrentdictionarytkeytvalue/ + while (true) + { + if (_values.TryGetValue(value, out var existingValue)) + { + Interlocked.Increment(ref _hitCount); + return existingValue; + } + + if (_values.TryAdd(value, value)) + { + Interlocked.Add(ref _charCount, value.Length); + return value; + } + } + } + + /// + /// Resets and back to zero. + /// + public void ResetCounts() + { + _requestCount = 0; + _hitCount = 0; + } + } +} + diff --git a/src/NuGet.Services.AzureSearch/AuxiliaryFiles/VerifiedPackagesDataClient.cs b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/VerifiedPackagesDataClient.cs new file mode 100644 index 000000000..929d105f6 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AuxiliaryFiles/VerifiedPackagesDataClient.cs @@ -0,0 +1,135 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Net; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Newtonsoft.Json; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class VerifiedPackagesDataClient : IVerifiedPackagesDataClient + { + private static readonly JsonSerializer Serializer = new JsonSerializer(); + + private readonly ICloudBlobClient _cloudBlobClient; + private readonly IOptionsSnapshot _options; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + private readonly Lazy _lazyContainer; + + public VerifiedPackagesDataClient( + ICloudBlobClient cloudBlobClient, + IOptionsSnapshot options, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _cloudBlobClient = cloudBlobClient ?? throw new ArgumentNullException(nameof(cloudBlobClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + _lazyContainer = new Lazy( + () => _cloudBlobClient.GetContainerReference(_options.Value.StorageContainer)); + } + + private ICloudBlobContainer Container => _lazyContainer.Value; + + public async Task>> ReadLatestAsync( + IAccessCondition accessCondition, + StringCache stringCache) + { + var stopwatch = Stopwatch.StartNew(); + var blobName = GetLatestIndexedBlobName(); + var blobReference = Container.GetBlobReference(blobName); + + _logger.LogInformation("Reading the latest verified packages from {BlobName}.", blobName); + + bool modified; + var data = new HashSet(StringComparer.OrdinalIgnoreCase); + AuxiliaryFileMetadata metadata; + try + { + using (var stream = await blobReference.OpenReadAsync(accessCondition)) + { + ReadStream(stream, id => data.Add(stringCache.Dedupe(id))); + modified = true; + metadata = new AuxiliaryFileMetadata( + lastModified: new DateTimeOffset(blobReference.LastModifiedUtc, TimeSpan.Zero), + loadDuration: stopwatch.Elapsed, + fileSize: blobReference.Properties.Length, + etag: blobReference.ETag); + } + } + catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotModified) + { + _logger.LogInformation("The blob {BlobName} has not changed.", blobName); + modified = false; + data = null; + metadata = null; + } + + stopwatch.Stop(); + _telemetryService.TrackReadLatestVerifiedPackages(data?.Count, modified, stopwatch.Elapsed); + + return new AuxiliaryFileResult>( + modified, + data, + metadata); + } + + public async Task ReplaceLatestAsync( + HashSet newData, + IAccessCondition accessCondition) + { + using (_telemetryService.TrackReplaceLatestVerifiedPackages(newData.Count)) + { + var blobName = GetLatestIndexedBlobName(); + _logger.LogInformation("Replacing the latest verified packages from {BlobName}.", blobName); + + var blobReference = Container.GetBlobReference(blobName); + + using (var stream = await blobReference.OpenWriteAsync(accessCondition)) + using (var streamWriter = new StreamWriter(stream)) + using (var jsonTextWriter = new JsonTextWriter(streamWriter)) + { + blobReference.Properties.ContentType = "application/json"; + Serializer.Serialize(jsonTextWriter, newData); + } + } + } + + private static void ReadStream(Stream stream, Action add) + { + using (var textReader = new StreamReader(stream)) + using (var jsonReader = new JsonTextReader(textReader)) + { + Guard.Assert(jsonReader.Read(), "The blob should be readable."); + Guard.Assert(jsonReader.TokenType == JsonToken.StartArray, "The first token should be the start of an array."); + Guard.Assert(jsonReader.Read(), "There should be a second token."); + while (jsonReader.TokenType == JsonToken.String) + { + var id = (string)jsonReader.Value; + add(id); + + Guard.Assert(jsonReader.Read(), "There should be a token after the string."); + } + + Guard.Assert(jsonReader.TokenType == JsonToken.EndArray, "The last token should be the end of an array."); + Guard.Assert(!jsonReader.Read(), "There should be no token after the end of the array."); + } + } + + private string GetLatestIndexedBlobName() + { + return $"{_options.Value.NormalizeStoragePath()}verified-packages/verified-packages.v1.json"; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/AzureSearchConfiguration.cs b/src/NuGet.Services.AzureSearch/AzureSearchConfiguration.cs new file mode 100644 index 000000000..16e3a82e5 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AzureSearchConfiguration.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch +{ + public class AzureSearchConfiguration + { + public string SearchServiceName { get; set; } + public string SearchServiceApiKey { get; set; } + public string SearchIndexName { get; set; } + public string HijackIndexName { get; set; } + public string StorageConnectionString { get; set; } + public string StorageContainer { get; set; } + public string StoragePath { get; set; } + public string FlatContainerBaseUrl { get; set; } + public string FlatContainerContainerName { get; set; } + public bool AllIconsInFlatContainer { get; set; } + + public string NormalizeStoragePath() + { + var storagePath = StoragePath?.Trim('/') ?? string.Empty; + if (storagePath.Length > 0) + { + storagePath = storagePath + "/"; + } + + return storagePath; + } + + public Uri ParseFlatContainerBaseUrl() + { + return new Uri(FlatContainerBaseUrl, UriKind.Absolute); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/AzureSearchException.cs b/src/NuGet.Services.AzureSearch/AzureSearchException.cs new file mode 100644 index 000000000..d0ce33489 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AzureSearchException.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch +{ + /// + /// An exception occurred while interacting with the Azure Search SDK. This exception is meant to be caught by an + /// exception filter in the web application so that HTTP status code 503 is returned to the user. The message in + /// this exception is not meant to become visible to the user. + /// + public class AzureSearchException : Exception + { + public AzureSearchException(string message, Exception innerException) + : base(message, innerException) + { + } + } +} diff --git a/src/NuGet.Services.AzureSearch/AzureSearchJob.cs b/src/NuGet.Services.AzureSearch/AzureSearchJob.cs new file mode 100644 index 000000000..d6fca6274 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AzureSearchJob.cs @@ -0,0 +1,55 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Net; +using System.Threading.Tasks; +using Autofac; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Rest; +using NuGet.Jobs; +using NuGet.Jobs.Validation; + +namespace NuGet.Services.AzureSearch +{ + public abstract class AzureSearchJob : JsonConfigurationJob where T : IAzureSearchCommand + { + private const string FeatureFlagConfigurationSectionName = "FeatureFlags"; + + public override async Task Run() + { + ServicePointManager.DefaultConnectionLimit = 64; + ServicePointManager.MaxServicePointIdleTime = 10000; + + var featureFlagRefresher = _serviceProvider.GetRequiredService(); + await featureFlagRefresher.StartIfConfiguredAsync(); + + var tracingInterceptor = _serviceProvider.GetRequiredService(); + try + { + ServiceClientTracing.IsEnabled = true; + ServiceClientTracing.AddTracingInterceptor(tracingInterceptor); + + await _serviceProvider.GetRequiredService().ExecuteAsync(); + } + finally + { + ServiceClientTracing.RemoveTracingInterceptor(tracingInterceptor); + } + + await featureFlagRefresher.StopAndWaitAsync(); + } + + protected override void ConfigureAutofacServices(ContainerBuilder containerBuilder, IConfigurationRoot configurationRoot) + { + containerBuilder.AddAzureSearch(); + } + + protected override void ConfigureJobServices(IServiceCollection services, IConfigurationRoot configurationRoot) + { + services.AddAzureSearch(GlobalTelemetryDimensions); + + services.Configure(configurationRoot.GetSection(FeatureFlagConfigurationSectionName)); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/AzureSearchJobConfiguration.cs b/src/NuGet.Services.AzureSearch/AzureSearchJobConfiguration.cs new file mode 100644 index 000000000..8a78b4d16 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AzureSearchJobConfiguration.cs @@ -0,0 +1,40 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch +{ + public class AzureSearchJobConfiguration : AzureSearchConfiguration + { + public int AzureSearchBatchSize { get; set; } = 1000; + + /// + /// The definition of batches is defined by the job that uses this configuration value, but in general this + /// property is used to control how many "workers" are allowed to work in parallel producing, and perhaps also + /// pushing, document changes for Azure Search. + /// + public int MaxConcurrentBatches { get; set; } = 4; + + /// + /// The maximum number of threads that should write version lists in parallel. This primary use for this + /// property is the implementation that updates version lists after all documents + /// for a specific package ID have been pushed to Azure Search. The specific semantics of this property vary + /// on a per-job basis. In some cases this property may be used within a batch + /// (i.e. ) so the actual maximum degree of parallelism for a single process + /// may be a multiple of this property. + /// + public int MaxConcurrentVersionListWriters { get; set; } = 8; + + public string GalleryBaseUrl { get; set; } + + public bool EnablePopularityTransfers { get; set; } + + public AzureSearchScoringConfiguration Scoring { get; set; } + + public Uri ParseGalleryBaseUrl() + { + return new Uri(GalleryBaseUrl, UriKind.Absolute); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/AzureSearchJobDevelopmentConfiguration.cs b/src/NuGet.Services.AzureSearch/AzureSearchJobDevelopmentConfiguration.cs new file mode 100644 index 000000000..9b8ac54ac --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AzureSearchJobDevelopmentConfiguration.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch +{ + public class AzureSearchJobDevelopmentConfiguration + { + /// + /// Disabling version lists writers speeds up db2azuresearch but breaks catalog2azuresearch. + /// This should be false on production environments. + /// + public bool DisableVersionListWriters { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/AzureSearchScoringConfiguration.cs b/src/NuGet.Services.AzureSearch/AzureSearchScoringConfiguration.cs new file mode 100644 index 000000000..da6ee58f5 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AzureSearchScoringConfiguration.cs @@ -0,0 +1,30 @@ +using System.Collections.Generic; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Configurations that control how package search results are scored for relevancy. + /// The index must be rebuilt after changing these values! + /// + public class AzureSearchScoringConfiguration + { + /// + /// Weights to increase the importance of matches on specific fields. The keys are + /// the names of the field whose weights should be modified, listed in . + /// The values are the weight of that field. + /// + public Dictionary FieldWeights { get; set; } + + /// + /// The percentage of downloads that should be transferred by the popularity transfer feature. + /// Values range from 0 to 1. + /// + public double PopularityTransfer { get; set; } + + /// + /// The magnitude boost. + /// This boosts packages with many downloads. + /// + public double DownloadScoreBoost { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/AzureSearchTelemetryService.cs b/src/NuGet.Services.AzureSearch/AzureSearchTelemetryService.cs new file mode 100644 index 000000000..99060f215 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/AzureSearchTelemetryService.cs @@ -0,0 +1,479 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; +using NuGet.Services.AzureSearch.SearchService; +using NuGet.Services.Logging; + +namespace NuGet.Services.AzureSearch +{ + public class AzureSearchTelemetryService : IAzureSearchTelemetryService + { + private const string Prefix = "AzureSearch."; + + private readonly ITelemetryClient _telemetryClient; + + public AzureSearchTelemetryService(ITelemetryClient telemetryClient) + { + _telemetryClient = telemetryClient ?? throw new ArgumentNullException(nameof(telemetryClient)); + } + + public IDisposable TrackVersionListsUpdated(int versionListCount, int workerCount) + { + return _telemetryClient.TrackDuration( + Prefix + "VersionListsUpdatedSeconds", + new Dictionary + { + { "VersionListCount", versionListCount.ToString() }, + { "WorkerCount", workerCount.ToString() }, + }); + } + + public void TrackIndexPushSuccess(string indexName, int documentCount, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "IndexPushSuccessSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "IndexName", indexName }, + { "DocumentCount", documentCount.ToString() }, + }); + } + + public void TrackIndexPushFailure(string indexName, int documentCount, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "IndexPushFailureSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "IndexName", indexName }, + { "DocumentCount", documentCount.ToString() }, + }); + } + + public void TrackIndexPushSplit(string indexName, int documentCount) + { + _telemetryClient.TrackMetric( + Prefix + "IndexPushSplit", + 1, + new Dictionary + { + { "IndexName", indexName }, + { "DocumentCount", documentCount.ToString() }, + }); + } + + public IDisposable TrackGetLatestLeaves(string packageId, int requestedVersions) + { + return _telemetryClient.TrackDuration( + Prefix + "GetLatestLeavesSeconds", + new Dictionary + { + { "PackageId", packageId }, + { "RequestVersions", requestedVersions.ToString() }, + }); + } + + public void TrackUpdateOwnersCompleted(JobOutcome outcome, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "UpdateOwnersCompletedSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "Outcome", outcome.ToString() }, + }); + } + + public void TrackAuxiliaryFilesReload(IReadOnlyList reloadedNames, IReadOnlyList notModifiedNames, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "AuxiliaryFilesReloadSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "ReloadedNames", JsonConvert.SerializeObject(reloadedNames) }, + { "NotModifiedNames", JsonConvert.SerializeObject(notModifiedNames) }, + }); + } + + public void TrackAuxiliaryFileDownloaded(string blobName, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "AuxiliaryFileDownloadedSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "BlobName", blobName }, + }); + } + + public void TrackGetOwnersForPackageId(int ownerCount, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "GetOwnersForPackageIdSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "OwnerCount", ownerCount.ToString() }, + }); + } + + public void TrackReadLatestIndexedOwners(int packageIdCount, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "ReadLatestIndexedOwnersSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "PackageIdCount", packageIdCount.ToString() }, + }); + } + + public IDisposable TrackUploadOwnerChangeHistory(int packageIdCount) + { + return _telemetryClient.TrackDuration( + Prefix + "UploadOwnerChangeHistorySeconds", + new Dictionary + { + { "PackageIdCount", packageIdCount.ToString() }, + }); + } + + public IDisposable TrackReplaceLatestIndexedOwners(int packageIdCount) + { + return _telemetryClient.TrackDuration( + Prefix + "ReplaceLatestIndexedOwnersSeconds", + new Dictionary + { + { "PackageIdCount", packageIdCount.ToString() }, + }); + } + + public void TrackReadLatestOwnersFromDatabase(int packageIdCount, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "ReadLatestOwnersFromDatabaseSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "PackageIdCount", packageIdCount.ToString() }, + }); + } + + public void TrackOwnerSetComparison(int oldCount, int newCount, int changeCount, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "OwnerSetComparisonSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "OldCount", oldCount.ToString() }, + { "NewCount", newCount.ToString() }, + { "ChangeCount", changeCount.ToString() }, + }); + } + + public void TrackReadLatestIndexedPopularityTransfers( + int? outgoingTransfers, + bool modified, + TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "ReadLatestIndexedPopularityTransfersSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "OutgoingTransfers", outgoingTransfers.ToString() }, + { "Modified", modified.ToString() } + }); + } + + public void TrackReadLatestPopularityTransfersFromDatabase(int outgoingTransfers, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "ReadLatestPopularityTransfersFromDatabase", + elapsed.TotalSeconds, + new Dictionary + { + { "OutgoingTransfers", outgoingTransfers.ToString() } + }); + } + + public void TrackPopularityTransfersSetComparison(int oldCount, int newCount, int changeCount, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "PopularityTransfersSetComparisonSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "OldCount", oldCount.ToString() }, + { "NewCount", oldCount.ToString() }, + { "ChangeCount", oldCount.ToString() }, + }); + } + + public IDisposable TrackReplaceLatestIndexedPopularityTransfers(int outogingTransfers) + { + return _telemetryClient.TrackDuration( + Prefix + "ReplaceLatestIndexedPopularityTransfers", + new Dictionary + { + { "OutgoingTransfers", outogingTransfers.ToString() } + }); + } + + public IDisposable TrackCatalog2AzureSearchProcessBatch(int catalogLeafCount, int latestCatalogLeafCount, int packageIdCount) + { + return _telemetryClient.TrackDuration( + Prefix + "Catalog2AzureSearchBatchSeconds", + new Dictionary + { + { "CatalogLeafCount", catalogLeafCount.ToString() }, + { "LatestCatalogLeafCount", latestCatalogLeafCount.ToString() }, + { "PackageIdCount", packageIdCount.ToString() }, + }); + } + + public void TrackV2SearchQueryWithSearchIndex(TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "V2SearchQueryWithSearchIndexMs", + elapsed.TotalMilliseconds); + } + + public void TrackV2SearchQueryWithHijackIndex(TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "V2SearchQueryWithHijackIndexMs", + elapsed.TotalMilliseconds); + } + + public void TrackAutocompleteQuery(TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "AutocompleteQueryMs", + elapsed.TotalMilliseconds); + } + + public void TrackV3SearchQuery(TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "V3SearchQueryMs", + elapsed.TotalMilliseconds); + } + + public void TrackGetSearchServiceStatus(SearchStatusOptions options, bool success, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "GetSearchServiceStatusMs", + elapsed.TotalMilliseconds, + new Dictionary + { + { "Options", options.ToString() }, + { "Success", success.ToString() }, + }); + } + + public void TrackDocumentCountQuery(string indexName, long count, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "DocumentCountQueryMs", + elapsed.TotalMilliseconds, + new Dictionary + { + { "IndexName", indexName }, + { "Count", count.ToString() }, + }); + } + + public void TrackWarmQuery(string indexName, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "WarmQueryMs", + elapsed.TotalMilliseconds, + new Dictionary + { + { "IndexName", indexName }, + }); + } + + public void TrackLastCommitTimestampQuery(string indexName, DateTimeOffset? lastCommitTimestamp, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "LastCommitTimestampQueryMs", + elapsed.TotalMilliseconds, + new Dictionary + { + { "IndexName", indexName }, + { "LastCommitTimestamp", lastCommitTimestamp?.ToString("O") }, + }); + } + + public void TrackReadLatestIndexedDownloads(int? packageIdCount, bool notModified, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "ReadLatestIndexedDownloadsSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "PackageIdCount", packageIdCount?.ToString() }, + { "NotModified", notModified.ToString() }, + }); + } + + public IDisposable TrackReplaceLatestIndexedDownloads(int packageIdCount) + { + return _telemetryClient.TrackDuration( + Prefix + "ReplaceLatestIndexedDownloadsSeconds", + new Dictionary + { + { "PackageIdCount", packageIdCount.ToString() }, + }); + } + + public void TrackDownloadSetComparison(int oldCount, int newCount, int changeCount, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "DownloadSetComparisonSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "OldCount", oldCount.ToString() }, + { "NewCount", newCount.ToString() }, + { "ChangeCount", changeCount.ToString() }, + }); + } + + public void TrackDownloadCountDecrease( + string packageId, + string version, + bool oldHasId, + bool oldHasVersion, + long oldDownloads, + bool newHasId, + bool newHasVersion, + long newDownloads) + { + _telemetryClient.TrackMetric( + Prefix + "DownloadCountDecrease", + 1, + new Dictionary + { + { "PackageId", packageId }, + { "Version", version }, + { "OldHasId", oldHasId.ToString() }, + { "OldHasVersion", oldHasVersion.ToString() }, + { "OldDownloads", oldDownloads.ToString() }, + { "NewHasId", newHasId.ToString() }, + { "NewHasVersion", newHasVersion.ToString() }, + { "NewDownloads", newDownloads.ToString() }, + }); + } + + public void TrackAuxiliary2AzureSearchCompleted(JobOutcome outcome, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "Auxiliary2AzureSearchCompletedSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "Outcome", outcome.ToString() }, + }); + } + + public void TrackV3GetDocument(TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "V3GetDocumentMs", + elapsed.TotalMilliseconds); + } + + public void TrackV2GetDocumentWithSearchIndex(TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "V2GetDocumentWithSearchIndexMs", + elapsed.TotalMilliseconds); + } + + public void TrackV2GetDocumentWithHijackIndex(TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "V2GetDocumentWithHijackIndexMs", + elapsed.TotalMilliseconds); + } + + public void TrackReadLatestVerifiedPackages(int? packageIdCount, bool modified, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "ReadLatestVerifiedPackagesSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "PackageIdCount", packageIdCount?.ToString() }, + { "Modified", modified.ToString() }, + }); + } + + public IDisposable TrackReplaceLatestVerifiedPackages(int packageIdCount) + { + return _telemetryClient.TrackDuration( + Prefix + "ReplaceLatestVerifiedPackagesSeconds", + new Dictionary + { + { "PackageIdCount", packageIdCount.ToString() }, + }); + } + + public void TrackAuxiliaryFilesStringCache(int stringCount, long charCount, int requestCount, int hitCount) + { + _telemetryClient.TrackMetric( + Prefix + "AuxiliaryFilesStringCache", + 1, + new Dictionary + { + { "StringCount", stringCount.ToString() }, + { "CharCount", charCount.ToString() }, + { "RequestCount", requestCount.ToString() }, + { "HitCount", hitCount.ToString() }, + }); + } + + public void TrackReadLatestVerifiedPackagesFromDatabase(int packageIdCount, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "ReadLatestVerifiedPackagesFromDatabaseSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "PackageIdCount", packageIdCount.ToString() }, + }); + } + + public void TrackUpdateVerifiedPackagesCompleted(JobOutcome outcome, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "UpdateVerifiedPackagesCompletedSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "Outcome", outcome.ToString() }, + }); + } + + public void TrackUpdateDownloadsCompleted(JobOutcome outcome, TimeSpan elapsed) + { + _telemetryClient.TrackMetric( + Prefix + "UpdateDownloadsCompletedSeconds", + elapsed.TotalSeconds, + new Dictionary + { + { "Outcome", outcome.ToString() }, + }); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/BaseDocumentBuilder.cs b/src/NuGet.Services.AzureSearch/BaseDocumentBuilder.cs new file mode 100644 index 000000000..0c23dc572 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/BaseDocumentBuilder.cs @@ -0,0 +1,319 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.Extensions.Options; +using NuGet.Frameworks; +using NuGet.Protocol.Catalog; +using NuGet.Services.Entities; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; +using NuGetGallery; +using PackageDependency = NuGet.Protocol.Catalog.PackageDependency; + +namespace NuGet.Services.AzureSearch +{ + public class BaseDocumentBuilder : IBaseDocumentBuilder + { + private static readonly VersionRangeFormatter VersionRangeFormatter = new VersionRangeFormatter(); + private static readonly DateTimeOffset UnlistedPublished = new DateTimeOffset(Metadata.Catalog.Constants.UnpublishedDate); + + private static readonly HashSet SpecialFrameworks = new HashSet + { + NuGetFramework.AnyFramework, + NuGetFramework.AgnosticFramework, + NuGetFramework.UnsupportedFramework + }; + + private readonly IOptionsSnapshot _options; + + public BaseDocumentBuilder(IOptionsSnapshot options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public void PopulateUpdated( + IUpdatedDocument document, + bool lastUpdatedFromCatalog) + { + document.SetLastUpdatedDocumentOnNextRead(); + document.LastDocumentType = document.GetType().FullName; + document.LastUpdatedFromCatalog = lastUpdatedFromCatalog; + } + + public void PopulateCommitted( + ICommittedDocument document, + bool lastUpdatedFromCatalog, + DateTimeOffset? lastCommitTimestamp, + string lastCommitId) + { + if (lastUpdatedFromCatalog) + { + if (lastCommitTimestamp == null) + { + throw new ArgumentNullException(nameof(lastCommitTimestamp)); + } + + if (lastCommitId == null) + { + throw new ArgumentNullException(nameof(lastCommitId)); + } + } + else + { + if (lastCommitTimestamp != null) + { + throw new ArgumentException("The last commit timestamp must be null when not updated from the catalog", nameof(lastCommitTimestamp)); + } + + if (lastCommitId != null) + { + throw new ArgumentException("The last commit ID must be null when not updated from the catalog", nameof(lastCommitId)); + } + } + + PopulateUpdated(document, lastUpdatedFromCatalog); + document.LastCommitTimestamp = lastCommitTimestamp; + document.LastCommitId = lastCommitId; + } + + public void PopulateMetadata( + IBaseMetadataDocument document, + string packageId, + Package package) + { + document.Authors = package.FlattenedAuthors; + document.Copyright = package.Copyright; + document.Created = AssumeUtc(package.Created); + document.Description = package.Description; + document.FileSize = package.PackageFileSize; + document.FlattenedDependencies = package.FlattenedDependencies; + document.Hash = package.Hash; + document.HashAlgorithm = package.HashAlgorithm; + document.Language = package.Language; + document.LastEdited = AssumeUtc(package.LastEdited); + document.MinClientVersion = package.MinClientVersion; + document.NormalizedVersion = package.NormalizedVersion; + document.OriginalVersion = package.Version; + document.PackageId = packageId; + document.Prerelease = package.IsPrerelease; + document.ProjectUrl = package.ProjectUrl; + document.Published = package.Listed ? AssumeUtc(package.Published) : UnlistedPublished; + document.ReleaseNotes = package.ReleaseNotes; + document.RequiresLicenseAcceptance = package.RequiresLicenseAcceptance; + document.SemVerLevel = package.SemVerLevelKey; + document.SortableTitle = GetSortableTitle(package.Title, packageId); + document.Summary = package.Summary; + document.Tags = package.Tags == null ? null : Utils.SplitTags(package.Tags); + document.Title = GetTitle(package.Title, packageId); + document.TokenizedPackageId = packageId; + + if (package.LicenseExpression != null || package.EmbeddedLicenseType != EmbeddedLicenseFileType.Absent) + { + document.LicenseUrl = LicenseHelper.GetGalleryLicenseUrl( + packageId, + package.NormalizedVersion, + _options.Value.ParseGalleryBaseUrl()); + } + else + { + document.LicenseUrl = package.LicenseUrl; + } + + if (package.HasEmbeddedIcon + || (!string.IsNullOrWhiteSpace(package.IconUrl) && _options.Value.AllIconsInFlatContainer)) + { + SetIconUrlFromFlatContainer(document); + } + else + { + document.IconUrl = package.IconUrl; + } + } + + public void PopulateMetadata( + IBaseMetadataDocument document, + string normalizedVersion, + PackageDetailsCatalogLeaf leaf) + { + document.Authors = leaf.Authors; + document.Copyright = leaf.Copyright; + document.Created = leaf.Created; + document.Description = leaf.Description; + document.FileSize = leaf.PackageSize; + document.FlattenedDependencies = GetFlattenedDependencies(leaf); + document.Hash = leaf.PackageHash; + document.HashAlgorithm = leaf.PackageHashAlgorithm; + document.Language = leaf.Language; + document.LastEdited = leaf.LastEdited; + document.MinClientVersion = leaf.MinClientVersion; + document.NormalizedVersion = normalizedVersion; + document.OriginalVersion = leaf.VerbatimVersion; + document.PackageId = leaf.PackageId; + document.Prerelease = leaf.IsPrerelease; + document.ProjectUrl = leaf.ProjectUrl; + document.Published = leaf.Published; + document.ReleaseNotes = leaf.ReleaseNotes; + document.RequiresLicenseAcceptance = leaf.RequireLicenseAcceptance; + document.SemVerLevel = leaf.IsSemVer2() ? SemVerLevelKey.SemVer2 : SemVerLevelKey.Unknown; + document.SortableTitle = GetSortableTitle(leaf.Title, leaf.PackageId); + document.Summary = leaf.Summary; + document.Tags = leaf.Tags == null ? null : leaf.Tags.ToArray(); + document.Title = GetTitle(leaf.Title, leaf.PackageId); + document.TokenizedPackageId = leaf.PackageId; + + if (leaf.LicenseExpression != null || leaf.LicenseFile != null) + { + document.LicenseUrl = LicenseHelper.GetGalleryLicenseUrl( + document.PackageId, + normalizedVersion, + _options.Value.ParseGalleryBaseUrl()); + } + else + { + document.LicenseUrl = leaf.LicenseUrl; + } + + if (leaf.IconFile != null + || (!string.IsNullOrWhiteSpace(leaf.IconUrl) && _options.Value.AllIconsInFlatContainer)) + { + SetIconUrlFromFlatContainer(document); + } + else + { + document.IconUrl = leaf.IconUrl; + } + } + + private void SetIconUrlFromFlatContainer(IBaseMetadataDocument document) + { + var provider = new FlatContainerPackagePathProvider(_options.Value.FlatContainerContainerName); + var iconPath = provider.GetIconPath(document.PackageId, document.NormalizedVersion); + document.IconUrl = new Uri(_options.Value.ParseFlatContainerBaseUrl(), iconPath).AbsoluteUri; + } + + private static string GetTitle(string title, string packageId) + { + return string.IsNullOrWhiteSpace(title) ? packageId : title; + } + + private static string GetSortableTitle(string title, string packageId) + { + var output = GetTitle(title, packageId); + return output.Trim().ToLowerInvariant(); + } + + private static DateTimeOffset? AssumeUtc(DateTime? dateTime) + { + if (!dateTime.HasValue) + { + return null; + } + + return new DateTimeOffset(dateTime.Value.Ticks, TimeSpan.Zero); + } + + /// + /// This method produces output for official client. The implementation on the client side, at one point of time + /// was: + /// https://github.com/NuGet/NuGet.Client/blob/b404acf6eb88c2b6086a9cbb5106104534de2428/src/NuGet.Core/NuGet.Protocol/LegacyFeed/V2FeedPackageInfo.cs#L228-L308 + /// + /// The output is a string where each dependency is separated by a "|" character and information about each + /// dependency is separated by a ":" character. Per dependency, the colon-seperated data is a pair or triple. + /// The three fields in order are: dependency package ID, version range, and target framework. If third value + /// (the framework that the dependency targets) is an empty string or excluded, this means the dependency + /// applies to any framework. If the package ID and range and empty strings but the framework is included, this + /// means that the package supports the specified framework but has no dependencies specific to that framework. + /// + /// Output: + /// [DEPENDENCY[|DEPENDENCY][|...][|DEPENDENCY]] + /// Each dependency: + /// [PACKAGE_ID]:[VERSION_RANGE][:TARGET_FRAMEWORK] + /// + /// Example A (no target frameworks): + /// Microsoft.Data.OData:5.0.2|Microsoft.WindowsAzure.ConfigurationManager:1.8.0 + /// Example B (target frameworks): + /// NETStandard.Library:1.6.1:netstandard1.0|Newtonsoft.Json:10.0.2:netstandard1.0 + /// Example D (empty target framework): + /// Microsoft.Data.OData:5.0.2: + /// Example D (just target framework): + /// ::net20|::net35|::net40|::net45|NETStandard.Library:1.6.1:netstandard1.0 + /// + private static string GetFlattenedDependencies(PackageDetailsCatalogLeaf leaf) + { + if (leaf.DependencyGroups == null) + { + return null; + } + + var builder = new StringBuilder(); + foreach (var dependencyGroup in leaf.DependencyGroups) + { + var targetFramework = dependencyGroup.ParseTargetFramework(); + + if (dependencyGroup.Dependencies != null && dependencyGroup.Dependencies.Any()) + { + foreach (var packageDependency in dependencyGroup.Dependencies) + { + AddFlattenedPackageDependency(targetFramework, packageDependency, builder); + } + } + else + { + if (builder.Length > 0) + { + builder.Append("|"); + } + + builder.Append(":"); + AddFlattenedFrameworkDependency(targetFramework, builder); + } + } + + return builder.Length > 0 ? builder.ToString() : null; + } + + private static void AddFlattenedPackageDependency( + NuGetFramework targetFramework, + PackageDependency packageDependency, + StringBuilder builder) + { + if (builder.Length > 0) + { + builder.Append("|"); + } + + builder.Append(packageDependency.Id); + builder.Append(":"); + + var versionRange = packageDependency.ParseRange(); + + if (!VersionRange.All.Equals(versionRange)) + { + builder.Append(versionRange?.ToString("S", VersionRangeFormatter)); + } + + AddFlattenedFrameworkDependency(targetFramework, builder); + } + + private static void AddFlattenedFrameworkDependency(NuGetFramework targetFramework, StringBuilder builder) + { + if (!SpecialFrameworks.Contains(targetFramework)) + { + try + { + builder.Append(":"); + builder.Append(targetFramework?.GetShortFolderName()); + } + catch (FrameworkException) + { + // ignoring FrameworkException on purpose - we don't want the job crashing + // whenever someone uploads an unsupported framework + } + } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/BatchPusher.cs b/src/NuGet.Services.AzureSearch/BatchPusher.cs new file mode 100644 index 000000000..898caa862 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/BatchPusher.cs @@ -0,0 +1,360 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using Microsoft.Azure.Search; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Rest.Azure; +using NuGet.Packaging; +using NuGet.Services.AzureSearch.Wrappers; + +namespace NuGet.Services.AzureSearch +{ + public class BatchPusher : IBatchPusher + { + private readonly ISearchIndexClientWrapper _searchIndexClient; + private readonly ISearchIndexClientWrapper _hijackIndexClient; + private readonly IVersionListDataClient _versionListDataClient; + private readonly IOptionsSnapshot _options; + private readonly IOptionsSnapshot _developmentOptions; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + internal readonly Dictionary _idReferenceCount; + internal readonly Queue>> _searchActions; + internal readonly Queue>> _hijackActions; + internal readonly Dictionary> _versionListDataResults; + + public BatchPusher( + ISearchIndexClientWrapper searchIndexClient, + ISearchIndexClientWrapper hijackIndexClient, + IVersionListDataClient versionListDataClient, + IOptionsSnapshot options, + IOptionsSnapshot developmentOptions, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _searchIndexClient = searchIndexClient ?? throw new ArgumentNullException(nameof(searchIndexClient)); + _hijackIndexClient = hijackIndexClient ?? throw new ArgumentNullException(nameof(hijackIndexClient)); + _versionListDataClient = versionListDataClient ?? throw new ArgumentNullException(nameof(versionListDataClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _developmentOptions = developmentOptions ?? throw new ArgumentNullException(nameof(developmentOptions)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _idReferenceCount = new Dictionary(StringComparer.OrdinalIgnoreCase); + + _searchActions = new Queue>>(); + _hijackActions = new Queue>>(); + _versionListDataResults = new Dictionary>(); + + if (_options.Value.MaxConcurrentVersionListWriters <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(options), + $"The {nameof(AzureSearchJobConfiguration.MaxConcurrentVersionListWriters)} must be greater than zero."); + } + + if (_options.Value.AzureSearchBatchSize <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(options), + $"The {nameof(AzureSearchJobConfiguration.AzureSearchBatchSize)} must be greater than zero."); + } + } + + public void EnqueueIndexActions(string packageId, IndexActions indexActions) + { + if (_versionListDataResults.ContainsKey(packageId)) + { + throw new ArgumentException("This package ID has already been enqueued.", nameof(packageId)); + } + + if (indexActions.IsEmpty) + { + throw new ArgumentException("There must be at least one index action.", nameof(indexActions)); + } + + foreach (var action in indexActions.Hijack) + { + EnqueueAndIncrement(_hijackActions, packageId, action); + } + + foreach (var action in indexActions.Search) + { + EnqueueAndIncrement(_searchActions, packageId, action); + } + + _versionListDataResults.Add(packageId, indexActions.VersionListDataResult); + } + + public async Task TryPushFullBatchesAsync() + { + return await TryPushBatchesAsync(onlyFull: true); + } + + public async Task TryFinishAsync() + { + return await TryPushBatchesAsync(onlyFull: false); + } + + private async Task TryPushBatchesAsync(bool onlyFull) + { + var failedPackageIds = new List(); + failedPackageIds.AddRange(await PushBatchesAsync(_hijackIndexClient, _hijackActions, onlyFull)); + failedPackageIds.AddRange(await PushBatchesAsync(_searchIndexClient, _searchActions, onlyFull)); + return new BatchPusherResult(failedPackageIds); + } + + private async Task> PushBatchesAsync( + ISearchIndexClientWrapper indexClient, + Queue>> actions, + bool onlyFull) + { + var failedPackageIds = new List(); + while ((onlyFull && actions.Count >= _options.Value.AzureSearchBatchSize) + || (!onlyFull && actions.Count > 0)) + { + var allFinished = new List>>(); + var batch = new List>(); + + while (batch.Count < _options.Value.AzureSearchBatchSize && actions.Count > 0) + { + var idAndValue = DequeueAndDecrement(actions, out int newCount); + batch.Add(idAndValue.Value); + + if (newCount == 0) + { + allFinished.Add(NewIdAndValue(idAndValue.Id, _versionListDataResults[idAndValue.Id])); + Guard.Assert(_versionListDataResults.Remove(idAndValue.Id), "The version list data result should have existed."); + } + } + + await IndexAsync(indexClient, batch); + + if (allFinished.Any()) + { + failedPackageIds.AddRange(await UpdateVersionListsAsync(allFinished)); + } + } + + Guard.Assert( + !_versionListDataResults + .Keys + .Except(_idReferenceCount.Keys) + .Any(), + "There are some version list data results without reference counts."); + Guard.Assert( + !_idReferenceCount + .Keys + .Except(_versionListDataResults.Keys) + .Any(), + "There are some reference counts without version list data results."); + + return failedPackageIds; + } + + private async Task IndexAsync( + ISearchIndexClientWrapper indexClient, + IReadOnlyCollection> batch) + { + if (batch.Count == 0) + { + return; + } + + if (batch.Count > _options.Value.AzureSearchBatchSize) + { + throw new ArgumentException("The provided batch is too large."); + } + + _logger.LogInformation( + "Pushing batch of {BatchSize} to index {IndexName}.", + batch.Count, + indexClient.IndexName); + + IList indexingResults = null; + Exception innerException = null; + var stopwatch = Stopwatch.StartNew(); + try + { + var batchResults = await indexClient.Documents.IndexAsync(new IndexBatch(batch)); + indexingResults = batchResults.Results; + + stopwatch.Stop(); + _telemetryService.TrackIndexPushSuccess(indexClient.IndexName, batch.Count, stopwatch.Elapsed); + } + catch (IndexBatchException ex) + { + stopwatch.Stop(); + _telemetryService.TrackIndexPushFailure(indexClient.IndexName, batch.Count, stopwatch.Elapsed); + + _logger.LogError( + 0, + ex, + "An exception was thrown while sending documents to index {IndexName}.", + indexClient.IndexName); + indexingResults = ex.IndexingResults; + innerException = ex; + } + catch (CloudException ex) when (ex.Response.StatusCode == HttpStatusCode.RequestEntityTooLarge && batch.Count > 1) + { + stopwatch.Stop(); + _telemetryService.TrackIndexPushSplit(indexClient.IndexName, batch.Count); + + var halfCount = batch.Count / 2; + var halfA = batch.Take(halfCount).ToList(); + var halfB = batch.Skip(halfCount).ToList(); + + _logger.LogWarning( + 0, + ex, + "The request body for a batch of {BatchSize} was too large. Splitting into two batches of size " + + "{HalfA} and {HalfB}.", + batch.Count, + halfA.Count, + halfB.Count); + + await IndexAsync(indexClient, halfA); + await IndexAsync(indexClient, halfB); + } + + if (indexingResults != null) + { + const int errorsToLog = 5; + var errorCount = 0; + foreach (var result in indexingResults) + { + if (!result.Succeeded) + { + if (errorCount < errorsToLog) + { + _logger.LogError( + "Indexing document with key {Key} failed for index {IndexName}. {StatusCode}: {ErrorMessage}", + result.Key, + indexClient.IndexName, + result.StatusCode, + result.ErrorMessage); + } + + errorCount++; + } + } + + if (errorCount > 0) + { + _logger.LogError( + "{ErrorCount} errors were found when indexing a batch for index {IndexName}. {LoggedErrors} were logged.", + errorCount, + indexClient.IndexName, + Math.Min(errorCount, errorsToLog)); + throw new InvalidOperationException( + $"Errors were found when indexing a batch. Up to {errorsToLog} errors get logged.", + innerException); + } + } + } + + private async Task> UpdateVersionListsAsync(List>> allFinished) + { + if (_developmentOptions.Value.DisableVersionListWriters) + { + _logger.LogWarning( + "Skipped updating {VersionListCount} version lists.", + allFinished.Count); + return new HashSet(StringComparer.OrdinalIgnoreCase); + } + + var versionListIdSample = allFinished + .OrderByDescending(x => x.Value.Result.VersionProperties.Count(v => v.Value.Listed)) + .Select(x => x.Id) + .Take(5) + .ToArray(); + var workerCount = Math.Min(allFinished.Count, _options.Value.MaxConcurrentVersionListWriters); + _logger.LogInformation( + "Updating {VersionListCount} version lists with {WorkerCount} workers, including {IdSample}.", + allFinished.Count, + workerCount, + versionListIdSample); + + var work = new ConcurrentBag>>(allFinished); + var failedPackageIds = new ConcurrentQueue(); + using (_telemetryService.TrackVersionListsUpdated(allFinished.Count, workerCount)) + { + var tasks = Enumerable + .Range(0, workerCount) + .Select(async x => + { + await Task.Yield(); + while (work.TryTake(out var finished)) + { + var success = await _versionListDataClient.TryReplaceAsync( + finished.Id, + finished.Value.Result, + finished.Value.AccessCondition); + + if (!success) + { + failedPackageIds.Enqueue(finished.Id); + } + } + }) + .ToList(); + await Task.WhenAll(tasks); + + _logger.LogInformation( + "Done updating {VersionListCount} version lists. {FailureCount} version lists failed.", + allFinished.Count - failedPackageIds.Count, + failedPackageIds.Count); + + return failedPackageIds.ToHashSet(StringComparer.OrdinalIgnoreCase); + } + } + + private void EnqueueAndIncrement(Queue> queue, string id, T value) + { + if (_idReferenceCount.TryGetValue(id, out var count)) + { + Guard.Assert(count >= 1, "The existing reference count should always be greater than zero."); + _idReferenceCount[id] = count + 1; + } + else + { + _idReferenceCount[id] = 1; + } + + queue.Enqueue(NewIdAndValue(id, value)); + } + + private IdAndValue DequeueAndDecrement(Queue> queue, out int newCount) + { + var idAndValue = queue.Dequeue(); + + var oldCount = _idReferenceCount[idAndValue.Id]; + newCount = oldCount - 1; + Guard.Assert(newCount >= 0, "The reference count should never be negative."); + + if (newCount == 0) + { + _idReferenceCount.Remove(idAndValue.Id); + } + else + { + _idReferenceCount[idAndValue.Id] = newCount; + } + + return idAndValue; + } + + private IdAndValue NewIdAndValue(string id, T value) + { + return new IdAndValue(id, value); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/BatchPusherResult.cs b/src/NuGet.Services.AzureSearch/BatchPusherResult.cs new file mode 100644 index 000000000..e9bc27dba --- /dev/null +++ b/src/NuGet.Services.AzureSearch/BatchPusherResult.cs @@ -0,0 +1,45 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace NuGet.Services.AzureSearch +{ + public class BatchPusherResult + { + public BatchPusherResult() : this(Array.Empty()) + { + } + + public BatchPusherResult(IEnumerable failedPackageIds) + { + FailedPackageIds = failedPackageIds.ToHashSet(StringComparer.OrdinalIgnoreCase); + } + + /// + /// Package IDs that failed due to access condition on the version list. + /// + public HashSet FailedPackageIds { get; } + + public bool Success => !FailedPackageIds.Any(); + + public void EnsureSuccess() + { + if (!Success) + { + throw new InvalidOperationException( + "The index operations for the following package IDs failed due to version list concurrency: " + + string.Join(", ", FailedPackageIds.OrderBy(x => x, StringComparer.OrdinalIgnoreCase))); + } + } + + public BatchPusherResult Merge(BatchPusherResult other) + { + return new BatchPusherResult(FailedPackageIds + .Concat(other.FailedPackageIds) + .ToHashSet(StringComparer.OrdinalIgnoreCase)); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/BlobContainerBuilder.cs b/src/NuGet.Services.AzureSearch/BlobContainerBuilder.cs new file mode 100644 index 000000000..45e5f20f2 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/BlobContainerBuilder.cs @@ -0,0 +1,115 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Net; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch +{ + public class BlobContainerBuilder : IBlobContainerBuilder + { + private readonly ICloudBlobClient _cloudBlobClient; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + private readonly Lazy _lazyContainer; + private readonly TimeSpan _retryDelay; + + public BlobContainerBuilder( + ICloudBlobClient cloudBlobClient, + IOptionsSnapshot options, + ILogger logger) : this( + cloudBlobClient, + options, + logger, + retryDelay: TimeSpan.FromSeconds(10)) + { + } + + /// + /// This constructor is used for testing. + /// + internal BlobContainerBuilder( + ICloudBlobClient cloudBlobClient, + IOptionsSnapshot options, + ILogger logger, + TimeSpan retryDelay) + { + _cloudBlobClient = cloudBlobClient ?? throw new ArgumentNullException(nameof(cloudBlobClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _lazyContainer = new Lazy(() => + { + return _cloudBlobClient.GetContainerReference(_options.Value.StorageContainer); + }); + _retryDelay = retryDelay; + } + + private ICloudBlobContainer Container => _lazyContainer.Value; + + public async Task CreateAsync(bool retryOnConflict) + { + _logger.LogInformation("Creating blob container {ContainerName}.", _options.Value.StorageContainer); + var containerCreated = false; + var waitStopwatch = Stopwatch.StartNew(); + while (!containerCreated) + { + try + { + await Container.CreateAsync(); + await Container.SetPermissionsAsync(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob }); + containerCreated = true; + } + catch (StorageException ex) when (retryOnConflict && ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict) + { + if (waitStopwatch.Elapsed < TimeSpan.FromMinutes(5)) + { + _logger.LogInformation( + "The blob container is still being deleted. Attempting creation again in {RetryDelay}.", + _retryDelay); + await Task.Delay(_retryDelay); + } + else + { + throw; + } + } + } + _logger.LogInformation("Done creating blob container {ContainerName}.", _options.Value.StorageContainer); + } + + public async Task CreateIfNotExistsAsync() + { + if (await Container.ExistsAsync(null, null)) + { + _logger.LogInformation("Skipping creation of blob container {ContainerName} since it already exists.", _options.Value.StorageContainer); + } + else + { + await CreateAsync(retryOnConflict: false); + } + } + + public async Task DeleteIfExistsAsync() + { + _logger.LogWarning("Attempting to delete blob container {ContainerName}.", _options.Value.StorageContainer); + var containerDeleted = await Container.DeleteIfExistsAsync(); + if (containerDeleted) + { + _logger.LogWarning("Done deleting blob container {ContainerName}.", _options.Value.StorageContainer); + } + else + { + _logger.LogInformation("Blob container {ContainerName} was not deleted since it does not exist.", _options.Value.StorageContainer); + } + + return containerDeleted; + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/AzureSearchCollectorLogic.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/AzureSearchCollectorLogic.cs new file mode 100644 index 000000000..ecc4a82f8 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/AzureSearchCollectorLogic.cs @@ -0,0 +1,192 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Protocol.Catalog; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.V3; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public class AzureSearchCollectorLogic : ICommitCollectorLogic + { + private readonly ICatalogIndexActionBuilder _indexActionBuilder; + private readonly Func _batchPusherFactory; + private readonly IDocumentFixUpEvaluator _fixUpEvaluator; + private readonly CommitCollectorUtility _utility; + private readonly IOptionsSnapshot _options; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + + public AzureSearchCollectorLogic( + ICatalogIndexActionBuilder indexActionBuilder, + Func batchPusherFactory, + IDocumentFixUpEvaluator fixUpEvaluator, + CommitCollectorUtility utility, + IOptionsSnapshot options, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _indexActionBuilder = indexActionBuilder ?? throw new ArgumentNullException(nameof(indexActionBuilder)); + _batchPusherFactory = batchPusherFactory ?? throw new ArgumentNullException(nameof(batchPusherFactory)); + _fixUpEvaluator = fixUpEvaluator ?? throw new ArgumentNullException(nameof(fixUpEvaluator)); + _utility = utility ?? throw new ArgumentNullException(nameof(utility)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (_options.Value.MaxConcurrentBatches <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(options), + $"The {nameof(AzureSearchJobConfiguration.MaxConcurrentBatches)} must be greater than zero."); + } + } + + public Task> CreateBatchesAsync(IEnumerable catalogItems) + { + // Create a single batch of all unprocessed catalog items so that we can have complete control of the + // parallelism in this class. + return Task.FromResult(_utility.CreateSingleBatch(catalogItems)); + } + + public async Task OnProcessBatchAsync(IEnumerable items) + { + var itemList = items.ToList(); + var attempt = 0; + var success = false; + while (!success) + { + attempt++; + var result = await ProcessItemsAsync(itemList, allowRetry: attempt < 3); + success = result.Success; + itemList = result.Items; + } + } + + /// + /// Processes the provided list of catalog items while handling known retriable errors. It is this method's + /// responsibility to throw an exception if the operation is unsuccessful and is + /// false, even if the failure is generally retriable. Failure to do so could lead to an infinite loop + /// in the caller. + /// + /// The item list to use for the Azure Search updates. + /// + /// False to make any error encountered bubble out as an exception. True if retriable errors should returned a + /// result with set to false. + /// + /// The result, including success boolean and the next item list to use for a retry. + private async Task ProcessItemsAsync(List itemList, bool allowRetry) + { + var latestItems = _utility.GetLatestPerIdentity(itemList); + var allWork = _utility.GroupById(latestItems); + + using (_telemetryService.TrackCatalog2AzureSearchProcessBatch(itemList.Count, latestItems.Count, allWork.Count)) + { + // In parallel, generate all index actions required to handle this batch. + var allIndexActions = await ProcessWorkAsync(latestItems, allWork); + + // In sequence, push batches of index actions to Azure Search. We do this because the maximum set of catalog + // items that can be processed here is a single catalog page, which has around 550 items. The maximum batch + // size for pushing to Azure Search is 1000 documents so there is no benefit to parallelizing this part. + // Azure Search indexing on their side is more efficient with fewer, larger batches. + var batchPusher = _batchPusherFactory(); + foreach (var indexAction in allIndexActions) + { + batchPusher.EnqueueIndexActions(indexAction.Id, indexAction.Value); + } + + try + { + var finishResult = await batchPusher.TryFinishAsync(); + if (allowRetry && !finishResult.Success) + { + _logger.LogWarning("Retrying catalog batch due to access condition failures on package IDs: {Ids}", finishResult.FailedPackageIds); + return new ProcessItemsResult(success: false, items: itemList); + } + + finishResult.EnsureSuccess(); + return new ProcessItemsResult(success: true, items: itemList); + } + catch (InvalidOperationException ex) when (allowRetry) + { + var result = await _fixUpEvaluator.TryFixUpAsync(itemList, allIndexActions, ex); + if (!result.Applicable) + { + throw; + } + + _logger.LogWarning("Retrying catalog batch due to Azure Search bug fix-up."); + return new ProcessItemsResult(success: false, items: result.ItemList); + } + } + } + + private class ProcessItemsResult + { + public ProcessItemsResult(bool success, List items) + { + Success = success; + Items = items ?? throw new ArgumentNullException(nameof(items)); + } + + public bool Success { get; } + + /// + /// The item list to used for the next iteration. + /// + public List Items { get; } + } + + private async Task>> ProcessWorkAsync( + IReadOnlyList latestItems, + ConcurrentBag>> allWork) + { + // Fetch the full catalog leaf for each item that is the package details type. + var allEntryToLeaf = await _utility.GetEntryToDetailsLeafAsync(latestItems); + + // Process the package ID groups in parallel, collecting all index actions for later. + var output = new ConcurrentBag>(); + var tasks = Enumerable + .Range(0, _options.Value.MaxConcurrentBatches) + .Select(async x => + { + await Task.Yield(); + while (allWork.TryTake(out var work)) + { + var entryToLeaf = work + .Value + .Where(CommitCollectorUtility.IsOnlyPackageDetails) + .ToDictionary(e => e, e => allEntryToLeaf[e], ReferenceEqualityComparer.Default); + var indexActions = await GetPackageIdIndexActionsAsync(work.Value, entryToLeaf); + output.Add(new IdAndValue(work.Id, indexActions)); + } + }) + .ToList(); + await Task.WhenAll(tasks); + + return output; + } + + private async Task GetPackageIdIndexActionsAsync( + IReadOnlyList entries, + IReadOnlyDictionary entryToLeaf) + { + var packageId = entries + .Select(x => x.PackageIdentity.Id) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Single(); + + return await _indexActionBuilder.AddCatalogEntriesAsync( + packageId, + entries, + entryToLeaf); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchCommand.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchCommand.cs new file mode 100644 index 000000000..d2d6c4db3 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchCommand.cs @@ -0,0 +1,111 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Services.V3; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public class Catalog2AzureSearchCommand : IAzureSearchCommand + { + public const string CursorRelativeUri = "cursor.json"; + + private readonly ICollector _collector; + private readonly IStorageFactory _storageFactory; + private readonly Func _handlerFunc; + private readonly IBlobContainerBuilder _blobContainerBuilder; + private readonly IIndexBuilder _indexBuilder; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public Catalog2AzureSearchCommand( + ICollector collector, + IStorageFactory storageFactory, + Func handlerFunc, + IBlobContainerBuilder blobContainerBuilder, + IIndexBuilder indexBuilder, + IOptionsSnapshot options, + ILogger logger) + { + _collector = collector ?? throw new ArgumentNullException(nameof(collector)); + _storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory)); + _handlerFunc = handlerFunc ?? throw new ArgumentNullException(nameof(handlerFunc)); + _blobContainerBuilder = blobContainerBuilder ?? throw new ArgumentNullException(nameof(blobContainerBuilder)); + _indexBuilder = indexBuilder ?? throw new ArgumentNullException(nameof(indexBuilder)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ExecuteAsync() + { + await ExecuteAsync(CancellationToken.None); + } + + private async Task ExecuteAsync(CancellationToken token) + { + // Initialize the cursors. + ReadCursor backCursor; + if (_options.Value.DependencyCursorUrls != null + && _options.Value.DependencyCursorUrls.Any()) + { + _logger.LogInformation("Depending on cursors:{DependencyCursorUrls}", _options.Value.DependencyCursorUrls); + backCursor = new AggregateCursor(_options + .Value + .DependencyCursorUrls.Select(r => new HttpReadCursor(new Uri(r), _handlerFunc))); + } + else + { + _logger.LogInformation("Depending on no cursors, meaning the job will process up to the latest catalog information."); + backCursor = MemoryCursor.CreateMax(); + } + + var frontCursorStorage = _storageFactory.Create(); + var frontCursorUri = frontCursorStorage.ResolveUri(CursorRelativeUri); + var frontCursor = new DurableCursor(frontCursorUri, frontCursorStorage, DateTime.MinValue); + + // Log information about where state will be kept. + _logger.LogInformation( + "Using storage URL: {ContainerUrl}/{StoragePath}", + CloudStorageAccount.Parse(_options.Value.StorageConnectionString) + .CreateCloudBlobClient() + .GetContainerReference(_options.Value.StorageContainer) + .Uri + .AbsoluteUri, + _options.Value.NormalizeStoragePath()); + _logger.LogInformation("Using cursor: {CursurUrl}", frontCursorUri.AbsoluteUri); + _logger.LogInformation("Using search service: {SearchServiceName}", _options.Value.SearchServiceName); + _logger.LogInformation("Using search index: {IndexName}", _options.Value.SearchIndexName); + _logger.LogInformation("Using hijack index: {IndexName}", _options.Value.HijackIndexName); + + // Optionally create the indexes. + if (_options.Value.CreateContainersAndIndexes) + { + await _blobContainerBuilder.CreateIfNotExistsAsync(); + await _indexBuilder.CreateSearchIndexIfNotExistsAsync(); + await _indexBuilder.CreateHijackIndexIfNotExistsAsync(); + } + + await frontCursor.LoadAsync(token); + await backCursor.LoadAsync(token); + _logger.LogInformation( + "The cursors have been loaded. Front: {FrontCursor}. Back: {BackCursor}.", + frontCursor.Value, + backCursor.Value); + + // Run the collector. + await _collector.RunAsync( + frontCursor, + backCursor, + token); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchConfiguration.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchConfiguration.cs new file mode 100644 index 000000000..ca37f2147 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/Catalog2AzureSearchConfiguration.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Services.V3; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public class Catalog2AzureSearchConfiguration : AzureSearchJobConfiguration, ICommitCollectorConfiguration + { + public int MaxConcurrentCatalogLeafDownloads { get; set; } = 64; + public bool CreateContainersAndIndexes { get; set; } + public string Source { get; set; } + public TimeSpan HttpClientTimeout { get; set; } = TimeSpan.FromMinutes(10); + public List DependencyCursorUrls { get; set; } + public string RegistrationsBaseUrl { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/CatalogIndexActionBuilder.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/CatalogIndexActionBuilder.cs new file mode 100644 index 000000000..92e6853a9 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/CatalogIndexActionBuilder.cs @@ -0,0 +1,507 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Logging; +using NuGet.Packaging.Core; +using NuGet.Protocol.Catalog; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public class CatalogIndexActionBuilder : ICatalogIndexActionBuilder + { + private static readonly int SearchFiltersCount = Enum.GetValues(typeof(SearchFilters)).Length; + + private readonly IVersionListDataClient _versionListDataClient; + private readonly ICatalogLeafFetcher _leafFetcher; + private readonly IDatabaseAuxiliaryDataFetcher _ownerFetcher; + private readonly ISearchDocumentBuilder _search; + private readonly IHijackDocumentBuilder _hijack; + private readonly ILogger _logger; + + public CatalogIndexActionBuilder( + IVersionListDataClient versionListDataClient, + ICatalogLeafFetcher leafFetcher, + IDatabaseAuxiliaryDataFetcher ownerFetcher, + ISearchDocumentBuilder search, + IHijackDocumentBuilder hijack, + ILogger logger) + { + _versionListDataClient = versionListDataClient ?? throw new ArgumentNullException(nameof(versionListDataClient)); + _leafFetcher = leafFetcher ?? throw new ArgumentNullException(nameof(leafFetcher)); + _ownerFetcher = ownerFetcher ?? throw new ArgumentNullException(nameof(ownerFetcher)); + _search = search ?? throw new ArgumentNullException(nameof(search)); + _hijack = hijack ?? throw new ArgumentNullException(nameof(hijack)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task AddCatalogEntriesAsync( + string packageId, + IReadOnlyList latestEntries, + IReadOnlyDictionary entryToLeaf) + { + if (latestEntries.Count == 0) + { + throw new ArgumentException("There must be at least one catalog item to process.", nameof(latestEntries)); + } + + var versionListDataResult = await _versionListDataClient.ReadAsync(packageId); + + var context = new Context( + packageId, + versionListDataResult, + latestEntries, + entryToLeaf); + + var indexChanges = await GetIndexChangesAsync(context); + + string[] owners; + if (indexChanges.Search.Values.Any(IsUpdateLatest)) + { + // Fetching owners is only strictly necessary on the AddFirst case. However, to facility reflow of + // owner information into the index, we also do this on UpdateLatest. We also do this on the + // DowngradeLatest case since it is a relatively uncommon case and is simpler to align with + // UpdateLatest. + owners = await _ownerFetcher.GetOwnersOrEmptyAsync(packageId); + } + else + { + // We fall into this case if the latest version of any search document is unchanged. The most likely + // example of this is if a user is unlisting an old version of the package. + owners = null; + } + + var search = indexChanges + .Search + .Select(p => GetSearchIndexAction( + context, + p.Key, + p.Value, + owners)) + .ToList(); + + var hijack = indexChanges + .Hijack + .Select(p => GetHijackIndexAction( + context, + p.Key, + p.Value)) + .ToList(); + + _logger.LogInformation( + "There are {SearchCount} search index changes and {HijackCount} hijack index changes for {PackageId}.", + search.Count, + hijack.Count, + packageId); + + return new IndexActions( + search, + hijack, + new ResultAndAccessCondition( + context.VersionLists.GetVersionListData(), + context.VersionListDataResult.AccessCondition)); + } + + private async Task GetIndexChangesAsync(Context context) + { + var downgradeLatest = new List(); + var versionsWithoutMetadata = new List(); + IndexChanges indexChanges; + var attempts = 0; + do + { + attempts++; + + if (versionsWithoutMetadata.Any()) + { + _logger.LogInformation( + "The following versions are now the latest on search documents, but have no metadata in the input catalog leafs: {Versions}", + versionsWithoutMetadata); + + var candidateVersionLists = downgradeLatest + .Select(x => x + .ListedFullVersions + .Select(NuGetVersion.Parse) + .ToList()) + .GroupBy(x => x, new CollectionComparer()) + .Select(x => x.First()) + .ToList(); + + var latestCatalogLeaves = await _leafFetcher.GetLatestLeavesAsync( + context.PackageId, + candidateVersionLists); + + foreach (var pair in latestCatalogLeaves.Available) + { + var entry = GetPackageDetailsEntry(pair.Key, pair.Value); + context.VersionToEntry[pair.Key] = entry; + context.EntryToLeaf[entry] = pair.Value; + } + + foreach (var unavailable in latestCatalogLeaves.Unavailable) + { + var entry = GetPackageDeleteEntry(context, unavailable); + context.VersionToEntry[unavailable] = entry; + context.EntryToLeaf.Remove(entry); + } + } + + var versionListChanges = context + .VersionToEntry + .Values + .Select(e => GetVersionListChange(context, e)) + .ToList(); + + context.VersionLists = new VersionLists(context.VersionListDataResult.Result); + + indexChanges = context.VersionLists.ApplyChanges(versionListChanges); + + downgradeLatest = indexChanges + .Search + .Where(x => x.Value == SearchIndexChangeType.DowngradeLatest) + .Select(x => context.VersionLists.GetLatestVersionInfoOrNull(x.Key)) + .ToList(); + versionsWithoutMetadata = downgradeLatest + .Where(x => !context.VersionToEntry.ContainsKey(x.ParsedVersion)) + .Select(x => x.ParsedVersion) + .OrderBy(x => x) + .Distinct() + .ToList(); + } + while (versionsWithoutMetadata.Any() && attempts < SearchFiltersCount); + + if (versionsWithoutMetadata.Any()) + { + const string message = "Too many attempts were made to fetch metadata for downgraded search documents."; + _logger.LogError( + message + " {Attempts} attempts were made for {PackageId}. Versions without metadata: {Versions}", + attempts, + context.PackageId, + versionsWithoutMetadata); + throw new InvalidOperationException(message); + } + + return indexChanges; + } + + private VersionListChange GetVersionListChange( + Context context, + CatalogCommitItem entry) + { + if (entry.IsPackageDetails && !entry.IsPackageDelete) + { + var leaf = context.EntryToLeaf[entry]; + return VersionListChange.Upsert( + leaf.VerbatimVersion ?? leaf.PackageVersion, + new VersionPropertiesData( + listed: leaf.IsListed(), + semVer2: leaf.IsSemVer2())); + } + else if (entry.IsPackageDelete && !entry.IsPackageDetails) + { + return VersionListChange.Delete(entry.PackageIdentity.Version); + } + else + { + const string message = "An unsupported leaf type was encountered."; + _logger.LogError( + message + " ID: {PackageId}, version: {PackageVersion}, commit timestamp: {CommitTimestamp:O}, " + + "types: {EntryTypeUris}, leaf URL: {Url}", + entry.PackageIdentity.Id, + entry.PackageIdentity.Version.ToFullString(), + entry.CommitTimeStamp, + entry.TypeUris, + entry.Uri.AbsoluteUri); + throw new ArgumentException("An unsupported leaf type was encountered."); + } + } + + private IndexAction GetSearchIndexAction( + Context context, + SearchFilters searchFilters, + SearchIndexChangeType changeType, + string[] owners) + { + var latestFlags = _search.LatestFlagsOrNull(context.VersionLists, searchFilters); + Guard.Assert( + changeType == SearchIndexChangeType.Delete || latestFlags != null, + "Either the search document is being or there is a latest version."); + + IndexAction indexAction; + + if (changeType == SearchIndexChangeType.Delete) + { + indexAction = IndexAction.Delete(_search.Keyed( + context.PackageId, + searchFilters)); + } + else if (changeType == SearchIndexChangeType.UpdateVersionList) + { + if (owners != null) + { + // If we have owner information already fetched on behalf of another search document, send the + // latest owner information as well. This provides two benefits: + // + // 1. This keeps all search documents for a package ID in-sync with regards to their owners + // fields. + // + // 2. This means if an admin is reflowing for the purposes of fixing up owner information, all + // search documents get the benefit instead of having to reflow the latest version of each + // search filter. + // + indexAction = IndexAction.Merge(_search.UpdateVersionListAndOwnersFromCatalog( + context.PackageId, + searchFilters, + lastCommitTimestamp: context.LastCommitTimestamp, + lastCommitId: context.LastCommitId, + versions: latestFlags.LatestVersionInfo.ListedFullVersions, + isLatestStable: latestFlags.IsLatestStable, + isLatest: latestFlags.IsLatest, + owners: owners)); + } + else + { + indexAction = IndexAction.Merge(_search.UpdateVersionListFromCatalog( + context.PackageId, + searchFilters, + lastCommitTimestamp: context.LastCommitTimestamp, + lastCommitId: context.LastCommitId, + versions: latestFlags.LatestVersionInfo.ListedFullVersions, + isLatestStable: latestFlags.IsLatestStable, + isLatest: latestFlags.IsLatest)); + } + } + else if (IsUpdateLatest(changeType)) + { + var leaf = context.GetLeaf(latestFlags.LatestVersionInfo.ParsedVersion); + var normalizedVersion = VerifyConsistencyAndNormalizeVersion(context, leaf); + indexAction = IndexAction.MergeOrUpload(_search.UpdateLatestFromCatalog( + searchFilters, + latestFlags.LatestVersionInfo.ListedFullVersions, + latestFlags.IsLatestStable, + latestFlags.IsLatest, + normalizedVersion, + latestFlags.LatestVersionInfo.FullVersion, + leaf, + owners)); + } + else + { + throw new NotImplementedException($"The change type '{changeType}' is not supported."); + } + + _logger.LogInformation( + "Search index action prepared for {PackageId} {SearchFilters}: {IndexAction} with a {DocumentType} document.", + context.PackageId, + searchFilters, + indexAction.ActionType, + indexAction.Document.GetType().FullName); + + return indexAction; + } + + /// + /// This is used to determine if a search index change type should be mapped to a + /// document. + /// + private bool IsUpdateLatest(SearchIndexChangeType changeType) + { + switch (changeType) + { + case SearchIndexChangeType.AddFirst: + case SearchIndexChangeType.UpdateLatest: + case SearchIndexChangeType.DowngradeLatest: + return true; + + default: + return false; + } + } + + private IndexAction GetHijackIndexAction( + Context context, + NuGetVersion version, + HijackDocumentChanges changes) + { + IndexAction indexAction; + + if (changes.Delete) + { + indexAction = IndexAction.Delete(_hijack.Keyed( + context.PackageId, + version.ToNormalizedString())); + } + else if (!changes.UpdateMetadata) + { + indexAction = IndexAction.Merge(_hijack.LatestFromCatalog( + context.PackageId, + version.ToNormalizedString(), + lastCommitTimestamp: context.LastCommitTimestamp, + lastCommitId: context.LastCommitId, + changes: changes)); + } + else + { + var leaf = context.GetLeaf(version); + var normalizedVersion = VerifyConsistencyAndNormalizeVersion(context, leaf); + + indexAction = IndexAction.MergeOrUpload(_hijack.FullFromCatalog( + normalizedVersion, + changes, + leaf)); + } + + _logger.LogInformation( + "Hijack index action prepared for {PackageId} {PackageVersion}: {IndexAction} with a {DocumentType} document.", + context.PackageId, + version.ToNormalizedString(), + indexAction.ActionType, + indexAction.Document.GetType().FullName); + + return indexAction; + } + + private string VerifyConsistencyAndNormalizeVersion( + Context context, + PackageDetailsCatalogLeaf leaf) + { + if (!StringComparer.OrdinalIgnoreCase.Equals(context.PackageId, leaf.PackageId)) + { + const string message = "The package ID found in the catalog package does not match the catalog leaf."; + _logger.LogError( + message + " Page ID: {PagePackageId}, leaf ID: {LeafPackageId}, leaf URL: {Url}", + context.PackageId, + leaf.PackageId, + leaf.Url); + throw new InvalidOperationException(message); + } + + var parsedPackageVersion = leaf.ParsePackageVersion(); + var normalizedVersion = parsedPackageVersion.ToNormalizedString(); + if (leaf.VerbatimVersion != null) + { + var parsedVerbatimVersion = NuGetVersion.Parse(leaf.VerbatimVersion); + if (normalizedVersion != parsedVerbatimVersion.ToNormalizedString()) + { + const string message = + "The normalized versions from the package version and the verbatim version do not match."; + _logger.LogError( + message + " ID: {PackageId}, version: {PackageVersion}, verbatim: {VerbatimVersion}, leaf URL: {Url}", + leaf.PackageId, + leaf.PackageVersion, + leaf.VerbatimVersion, + leaf.Url); + throw new InvalidOperationException(message); + } + } + + if (parsedPackageVersion.IsPrerelease != leaf.IsPrerelease) + { + var message = + $"The {nameof(PackageDetailsCatalogLeaf.IsPrerelease)} from the leaf does not match the version. " + + $"Using the value from the parsed version. "; + _logger.LogWarning( + message + " ID: {PackageId}, version: {PackageVersion}, leaf is prerelease: {LeafIsPrerelease}, " + + "parsed is prerelease: {ParsedIsPrerelease}, leaf URL: {Url}", + leaf.PackageId, + leaf.PackageVersion, + leaf.IsPrerelease, + parsedPackageVersion.IsPrerelease, + leaf.Url); + leaf.IsPrerelease = parsedPackageVersion.IsPrerelease; + } + + return normalizedVersion; + } + + private CatalogCommitItem GetPackageDetailsEntry(NuGetVersion version, PackageDetailsCatalogLeaf leaf) + { + return new CatalogCommitItem( + uri: new Uri(leaf.Url, UriKind.Absolute), + commitId: leaf.CommitId, + commitTimeStamp: leaf.CommitTimestamp.UtcDateTime, + types: new string[0], + typeUris: new[] { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity(leaf.PackageId, version)); + } + + private CatalogCommitItem GetPackageDeleteEntry(Context context, NuGetVersion version) + { + return new CatalogCommitItem( + uri: null, + commitId: null, + commitTimeStamp: DateTime.MinValue, + types: new string[0], + typeUris: new[] { Schema.DataTypes.PackageDelete }, + packageIdentity: new PackageIdentity(context.PackageId, version)); + } + + private class Context + { + public Context( + string packageId, + ResultAndAccessCondition versionListDataResult, + IEnumerable latestEntries, + IReadOnlyDictionary entryToLeaf) + { + PackageId = packageId; + VersionListDataResult = versionListDataResult; + VersionToEntry = latestEntries.ToDictionary(x => x.PackageIdentity.Version); + EntryToLeaf = entryToLeaf.ToDictionary( + x => x.Key, + x => x.Value, + ReferenceEqualityComparer.Default); + + var lastCommit = latestEntries + .GroupBy(x => new { x.CommitTimeStamp, x.CommitId }) + .Select(x => x.Key) + .OrderByDescending(x => x.CommitTimeStamp) + .First(); + + // Assume UTC on the commit timestamp. + LastCommitTimestamp = new DateTimeOffset(lastCommit.CommitTimeStamp.Ticks, TimeSpan.Zero); + LastCommitId = lastCommit.CommitId; + } + + public string PackageId { get; } + public ResultAndAccessCondition VersionListDataResult { get; } + public Dictionary VersionToEntry { get; } + public Dictionary EntryToLeaf { get; } + public VersionLists VersionLists { get; set; } + public DateTimeOffset LastCommitTimestamp { get; } + public string LastCommitId { get; } + + public PackageDetailsCatalogLeaf GetLeaf(NuGetVersion version) + { + var entry = VersionToEntry[version]; + if (entry.IsPackageDelete) + { + throw new ArgumentException("Leaves are not fetched for deleted versions.", nameof(version)); + } + + return EntryToLeaf[entry]; + } + } + + private class CollectionComparer : IEqualityComparer> + { + public bool Equals(IReadOnlyCollection x, IReadOnlyCollection y) + { + return x.SequenceEqual(y); + } + + public int GetHashCode(IReadOnlyCollection obj) + { + return obj + .OrderBy(x => x) + .Aggregate(0, (sum, i) => unchecked(sum + (i?.GetHashCode() ?? 0))); + } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/CatalogLeafFetcher.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/CatalogLeafFetcher.cs new file mode 100644 index 000000000..c8002854e --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/CatalogLeafFetcher.cs @@ -0,0 +1,318 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Protocol.Catalog; +using NuGet.Protocol.Registration; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public class CatalogLeafFetcher : ICatalogLeafFetcher + { + private readonly IRegistrationClient _registrationClient; + private readonly ICatalogClient _catalogClient; + private readonly IOptionsSnapshot _options; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + + public CatalogLeafFetcher( + IRegistrationClient registrationClient, + ICatalogClient catalogClient, + IOptionsSnapshot options, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _registrationClient = registrationClient ?? throw new ArgumentNullException(nameof(registrationClient)); + _catalogClient = catalogClient ?? throw new ArgumentNullException(nameof(catalogClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (_options.Value.MaxConcurrentBatches <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(options), + $"The {nameof(AzureSearchJobConfiguration.MaxConcurrentBatches)} must be greater than zero."); + } + + if (_options.Value.RegistrationsBaseUrl == null) + { + throw new ArgumentException( + $"The {nameof(Catalog2AzureSearchConfiguration.RegistrationsBaseUrl)} must be set.", + nameof(options)); + } + } + + public async Task GetLatestLeavesAsync( + string packageId, + IReadOnlyList> versions) + { + if (packageId == null) + { + throw new ArgumentNullException(nameof(packageId)); + } + + if (versions == null) + { + throw new ArgumentNullException(nameof(versions)); + } + + if (!versions.Any()) + { + throw new ArgumentException("At least one version list must be provided.", nameof(versions)); + } + + using (_telemetryService.TrackGetLatestLeaves(packageId, versions.SelectMany(v => v).Distinct().Count())) + { + var unavailable = new HashSet(); + var fetched = new Dictionary(); + var unlisted = new Dictionary(); + + var registrationIndexUrl = RegistrationUrlBuilder.GetIndexUrl(_options.Value.RegistrationsBaseUrl, packageId); + var registrationIndex = await _registrationClient.GetIndexOrNullAsync(registrationIndexUrl); + if (registrationIndex == null) + { + _logger.LogWarning( + "No registation index was found. ID: {PackageId}, registration index URL: {RegistrationIndexUrl}", + packageId, + registrationIndexUrl); + + foreach (var version in versions.SelectMany(x => x)) + { + unavailable.Add(version); + } + + return new LatestCatalogLeaves(unavailable, fetched); + } + + var pageUrlToInfo = registrationIndex + .Items + .ToDictionary(x => x.Url, x => new RegistrationPageInfo(x)); + + // Make a list of ranges for logging purposes. + var ranges = pageUrlToInfo + .OrderBy(x => x.Value.Range.MinVersion) + .Select(x => x.Value.RangeString) + .ToList(); + + foreach (var versionList in versions) + { + await AddLatestLeafAsync( + packageId, + versionList, + pageUrlToInfo, + ranges, + unavailable, + fetched, + unlisted); + } + + // Fetch the unlisted version's metadata in parallel. This can be many versions in the case of a bulk + // unlist. + var allWork = new ConcurrentBag>(unlisted); + var allResults = new ConcurrentBag>(); + var tasks = Enumerable + .Range(0, _options.Value.MaxConcurrentBatches) + .Select(async x => + { + await Task.Yield(); + while (allWork.TryTake(out var work)) + { + var leaf = await _catalogClient.GetPackageDetailsLeafAsync(work.Value); + allResults.Add(KeyValuePair.Create(work.Key, leaf)); + } + }) + .ToList(); + await Task.WhenAll(tasks); + foreach (var pair in allResults) + { + fetched.Add(pair.Key, pair.Value); + } + + return new LatestCatalogLeaves(unavailable, fetched); + } + } + + private async Task AddLatestLeafAsync( + string packageId, + IReadOnlyList versionList, + Dictionary pageUrlToInfo, + List ranges, + HashSet unavailable, + Dictionary fetched, + Dictionary unlisted) + { + var descendingVersions = versionList + .OrderByDescending(x => x) + .ToList(); + + foreach (var version in descendingVersions) + { + if (unavailable.Contains(version)) + { + _logger.LogDebug( + "For {PackageId}, version {Version} was already discovered to be unavailable.", + packageId, + version); + continue; + } + + if (unlisted.ContainsKey(version)) + { + _logger.LogDebug( + "For {PackageId}, version {Version} was already discovered to be unlisted.", + packageId, + version); + continue; + } + + if (fetched.TryGetValue(version, out var leaf)) + { + _logger.LogDebug( + "For {PackageId}, version {Version} was already fetched.", + packageId, + version); + } + else + { + _logger.LogInformation( + "Looking for the catalog leaf for {PackageId} {Version}.", + packageId, + version); + + var info = GetPageInfo(pageUrlToInfo, version); + if (info == null) + { + _logger.LogWarning( + "No page was found for {PackageId} {Version}. Page ranges were: {Ranges}", + packageId, + version, + ranges); + unavailable.Add(version); + continue; + } + + // When the items are not inlined, we need to make a network request to get the metadata. + if (info.VersionToItem == null) + { + _logger.LogInformation( + "Fetching the items for page {PageUrl}. Range: {Range}", + info.Page.Url, + info.RangeString); + + var page = await _registrationClient.GetPageAsync(info.Page.Url); + info.SetVersionToItem(page.Items); + } + + if (!info.VersionToItem.TryGetValue(version, out var item)) + { + _logger.LogWarning( + "No registration leaf item found for {PackageId} {Version} on {PageUrl}", + packageId, + version, + info.Page.Url); + unavailable.Add(version); + continue; + } + + if (!item.CatalogEntry.Listed) + { + _logger.LogInformation( + "{PackageId} {Version} was found to be unlisted from page {Url}. This will not be used as a latest version.", + packageId, + version, + info.Page.Url); + unlisted.Add(version, item.CatalogEntry.Url); + continue; + } + + _logger.LogInformation( + "Fetching the catalog leaf for {PackageId} {Version} from {LeafUrl}", + packageId, + version, + item.CatalogEntry.Url); + + leaf = await _catalogClient.GetPackageDetailsLeafAsync(item.CatalogEntry.Url); + fetched.Add(version, leaf); + } + + if (leaf.IsListed()) + { + _logger.LogInformation( + "{PackageId} {Version} was found to be listed. Metadata from {Url} will be used.", + packageId, + version, + leaf.Url); + return; + } + else + { + // We'll only hit this case if the catalog index/page told us that this version was listed but the + // leaf says that it is unlisted. + _logger.LogInformation( + "{PackageId} {Version} was found to be unlisted from leaf {Url}. This will not be used as a latest version.", + packageId, + version, + leaf.Url); + } + } + + _logger.LogWarning( + "No catalog leaves matchings leaves found for {PackageId}. Versions tried: {Versions}", + packageId, + descendingVersions); + } + + private RegistrationPageInfo GetPageInfo( + IReadOnlyDictionary pageUrlToInfo, + NuGetVersion version) + { + foreach (var info in pageUrlToInfo.Values) + { + if (info.Range.Satisfies(version)) + { + return info; + } + } + + return null; + } + + private class RegistrationPageInfo + { + public RegistrationPageInfo(RegistrationPage page) + { + Page = page; + Range = new VersionRange( + minVersion: NuGetVersion.Parse(page.Lower), + includeMinVersion: true, + maxVersion: NuGetVersion.Parse(page.Upper), + includeMaxVersion: true); + RangeString = Range.ToNormalizedString(); + + // When the items are inlined, we don't need to make a network request to get the metadata. + if (page.Items != null) + { + SetVersionToItem(page.Items); + } + } + + public void SetVersionToItem(IEnumerable items) + { + VersionToItem = items.ToDictionary(x => NuGetVersion.Parse(x.CatalogEntry.Version)); + } + + public RegistrationPage Page { get; } + public VersionRange Range { get; } + public string RangeString { get; } + public Dictionary VersionToItem { get; private set; } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/DocumentFixUp.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/DocumentFixUp.cs new file mode 100644 index 000000000..fce3b6d5f --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/DocumentFixUp.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Services.Metadata.Catalog; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public class DocumentFixUp + { + private DocumentFixUp(bool applicable, List itemList) + { + Applicable = applicable; + ItemList = itemList; + } + + public static DocumentFixUp IsNotApplicable() + { + return new DocumentFixUp(applicable: false, itemList: null); + } + + public static DocumentFixUp IsApplicable(List itemList) + { + if (itemList == null) + { + throw new ArgumentNullException(nameof(itemList)); + } + + return new DocumentFixUp(applicable: true, itemList: itemList); + } + + public bool Applicable { get; } + public List ItemList { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/DocumentFixUpEvaluator.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/DocumentFixUpEvaluator.cs new file mode 100644 index 000000000..e9a23c5d1 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/DocumentFixUpEvaluator.cs @@ -0,0 +1,164 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using Microsoft.Azure.Search; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Logging; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public class DocumentFixUpEvaluator : IDocumentFixUpEvaluator + { + private readonly IVersionListDataClient _versionListClient; + private readonly ICatalogLeafFetcher _leafFetcher; + private readonly ILogger _logger; + + public DocumentFixUpEvaluator( + IVersionListDataClient versionListClient, + ICatalogLeafFetcher leafFetcher, + ILogger logger) + { + _versionListClient = versionListClient ?? throw new ArgumentNullException(nameof(versionListClient)); + _leafFetcher = leafFetcher ?? throw new ArgumentNullException(nameof(leafFetcher)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task TryFixUpAsync( + IReadOnlyList itemList, + ConcurrentBag> allIndexActions, + InvalidOperationException exception) + { + var innerEx = exception.InnerException as IndexBatchException; + if (innerEx == null || innerEx.IndexingResults == null) + { + return DocumentFixUp.IsNotApplicable(); + } + + // There may have been a Case of the Missing Document! We have confirmed with the Azure Search team that + // this is a bug on the Azure Search side. To mitigate the issue, we replace any Merge operation that + // failed with 404 with a MergeOrUpload with the full metadata so that we can replace that missing document. + // + // 1. The first step is to find all of the document keys that failed with a 404 Not Found error. + var notFoundKeys = new HashSet(innerEx + .IndexingResults + .Where(x => x.StatusCode == (int)HttpStatusCode.NotFound) + .Select(x => x.Key)); + if (!notFoundKeys.Any()) + { + return DocumentFixUp.IsNotApplicable(); + } + + _logger.LogWarning("{Count} document action(s) failed with 404 Not Found.", notFoundKeys.Count); + + // 2. Find all of the package IDs that were affected, only considering Merge operations against the Search + // index. We ignore the the hijack index for now because we have only ever seen the problem in the Search + // index. + var failedIds = new HashSet(); + foreach (var pair in allIndexActions.OrderBy(x => x.Id, StringComparer.OrdinalIgnoreCase)) + { + var failedMerges = pair + .Value + .Search + .Where(a => a.ActionType == IndexActionType.Merge) + .Where(a => notFoundKeys.Contains(a.Document.Key)); + + if (failedMerges.Any() && failedIds.Add(pair.Id)) + { + _logger.LogWarning("Package {PackageId} had a Merge operation fail with 404 Not Found.", pair.Id); + } + } + + if (!failedIds.Any()) + { + _logger.LogInformation("No failed Merge operations against the Search index were found."); + return DocumentFixUp.IsNotApplicable(); + } + + // 3. For each affected package ID, get the version list to determine the latest version per search filter + // so we can find the the catalog entry for the version. + var identityToItems = itemList.GroupBy(x => x.PackageIdentity).ToDictionary(x => x.Key, x => x.ToList()); + foreach (var packageId in failedIds) + { + var accessConditionAndData = await _versionListClient.ReadAsync(packageId); + var versionLists = new VersionLists(accessConditionAndData.Result); + + var latestVersions = DocumentUtilities + .AllSearchFilters + .Select(sf => versionLists.GetLatestVersionInfoOrNull(sf)) + .Where(lvi => lvi != null) + .Select(lvi => (IReadOnlyList)new List { lvi.ParsedVersion }) + .ToList(); + + var leaves = await _leafFetcher.GetLatestLeavesAsync(packageId, latestVersions); + + // We ignore unavailable (deleted) versions for now. We have never had a delete cause this problem. It's + // only ever been discovered when a new version is being added or updated. + // + // For each package details leaf found, create a catalog commit item and add it to the set of items we + // will process. This will force the metadata to be updated on each of the latest versions. Since this + // is the latest metadata, replace any older leaves that may be associated with that package version. + foreach (var pair in leaves.Available) + { + var identity = new PackageIdentity(packageId, pair.Key); + var leaf = pair.Value; + + if (identityToItems.TryGetValue(identity, out var existing)) + { + if (existing.Count == 1 && existing[0].Uri.AbsoluteUri == leaf.Url) + { + _logger.LogInformation( + "For {PackageId} {PackageVersion}, metadata will remain the same.", + identity.Id, + identity.Version.ToNormalizedString(), + leaf.Url, + existing.Count); + continue; + } + else + { + _logger.LogInformation( + "For {PackageId} {PackageVersion}, metadata from {Url} will be used instead of {Count} catalog commit items.", + identity.Id, + identity.Version.ToNormalizedString(), + leaf.Url, + existing.Count); + } + } + else + { + _logger.LogInformation( + "For {PackageId} {PackageVersion}, metadata from {Url} will be used.", + identity.Id, + identity.Version.ToNormalizedString(), + leaf.Url); + } + + identityToItems[identity] = new List + { + new CatalogCommitItem( + new Uri(leaf.Url), + leaf.CommitId, + leaf.CommitTimestamp.UtcDateTime, + new string[0], + new[] { Schema.DataTypes.PackageDetails }, + identity), + }; + } + } + + _logger.LogInformation("The catalog commit item list has been modified to fix up the missing document(s)."); + + var newItemList = identityToItems.SelectMany(x => x.Value).ToList(); + return DocumentFixUp.IsApplicable(newItemList); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ICatalogIndexActionBuilder.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ICatalogIndexActionBuilder.cs new file mode 100644 index 000000000..3be24ccd3 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ICatalogIndexActionBuilder.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGet.Protocol.Catalog; +using NuGet.Services.Metadata.Catalog; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public interface ICatalogIndexActionBuilder + { + Task AddCatalogEntriesAsync( + string packageId, + IReadOnlyList latestEntries, + IReadOnlyDictionary entryToLeaf); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ICatalogLeafFetcher.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ICatalogLeafFetcher.cs new file mode 100644 index 000000000..9b86541f1 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/ICatalogLeafFetcher.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public interface ICatalogLeafFetcher + { + /// + /// Fetch information about the latest versions available in the catalog, via the package metadata + /// (registration) resource. At least one version in each provided version list is returned in the + /// property of the result, assuming there is are + /// any available versions. Versions referenced in the input version lists that could have been the new latest + /// version but are deleted will appear in . Versions not + /// referenced in the version lists but available in the registration index will be ignored. + /// + /// The whole purpose of this class is to handle the case + /// where we need to put a lower version's metadata in the search index document but we don't have that metadata + /// available. For example, this happens when a single catalog leaf comes in unlisting the currently latest + /// version. + /// + /// The input is a list of lists because multiple downgrades can happen for a single package ID. For example, + /// consider the latest version was 3.0.0 and the other versions are 1.0.0 and 2.0.0-beta. If 3.0.0 is unlisted + /// then 1.0.0 is the latest version for but 2.0.0-beta is the latest + /// version for . There can be as many input version lists as there + /// are different values. + /// + /// The package ID to fetch catalog leaves for. + /// The list of candidate latest versions. + /// The latest catalog leaves. + Task GetLatestLeavesAsync(string packageId, IReadOnlyList> versions); + } +} diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/IDocumentFixUpEvaluator.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/IDocumentFixUpEvaluator.cs new file mode 100644 index 000000000..939a76c1f --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/IDocumentFixUpEvaluator.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public interface IDocumentFixUpEvaluator + { + Task TryFixUpAsync( + IReadOnlyList itemList, + ConcurrentBag> allIndexActions, + InvalidOperationException exception); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/LatestCatalogLeaves.cs b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/LatestCatalogLeaves.cs new file mode 100644 index 000000000..731d5dad2 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Catalog2AzureSearch/LatestCatalogLeaves.cs @@ -0,0 +1,24 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Protocol.Catalog; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public class LatestCatalogLeaves + { + public LatestCatalogLeaves( + ISet unavailable, + IReadOnlyDictionary available) + { + Unavailable = unavailable ?? throw new ArgumentNullException(nameof(unavailable)); + Available = available ?? throw new ArgumentNullException(nameof(available)); + } + + public ISet Unavailable { get; } + public IReadOnlyDictionary Available { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/DatabaseAuxiliaryDataFetcher.cs b/src/NuGet.Services.AzureSearch/DatabaseAuxiliaryDataFetcher.cs new file mode 100644 index 000000000..5ff1c47b3 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/DatabaseAuxiliaryDataFetcher.cs @@ -0,0 +1,232 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Entity; +using System.Data.SqlClient; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Jobs; +using NuGet.Jobs.Configuration; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.AzureSearch +{ + public class DatabaseAuxiliaryDataFetcher : IDatabaseAuxiliaryDataFetcher + { + private readonly ISqlConnectionFactory _connectionFactory; + private readonly IEntitiesContextFactory _entitiesContextFactory; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + + private const int SqlCommandTimeoutSeconds = 120; + + private const string GetVerifiedPackagesSql = @" +SELECT pr.Id +FROM PackageRegistrations pr (NOLOCK) +WHERE pr.IsVerified = 1 +"; + + private const string GetPackageIdToOwnersSql = @" +SELECT + pr.Id, + u.Username +FROM PackageRegistrations pr (NOLOCK) +INNER JOIN PackageRegistrationOwners pro (NOLOCK) ON pro.PackageRegistrationKey = pr.[Key] +INNER JOIN Users u (NOLOCK) ON pro.UserKey = u.[Key] +"; + + private const int GetPopularityTransfersPageSize = 1000; + private const string GetPopularityTransfersSkipParameter = "@skip"; + private const string GetPopularityTransfersTakeParameter = "@take"; + private const string GetPopularityTransfersSql = @" +SELECT TOP (@take) + fpr.Id AS FromPackageId, + tpr.Id AS ToPackageId +FROM PackageRenames r (NOLOCK) +INNER JOIN PackageRegistrations fpr (NOLOCK) ON fpr.[Key] = r.[FromPackageRegistrationKey] +INNER JOIN PackageRegistrations tpr (NOLOCK) ON tpr.[Key] = r.[ToPackageRegistrationKey] +WHERE r.TransferPopularity != 0 AND r.[Key] >= @skip +ORDER BY r.[Key] ASC +"; + + public DatabaseAuxiliaryDataFetcher( + ISqlConnectionFactory connectionFactory, + IEntitiesContextFactory entitiesContextFactory, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); + _entitiesContextFactory = entitiesContextFactory ?? throw new ArgumentNullException(nameof(entitiesContextFactory)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger; + } + + public async Task GetOwnersOrEmptyAsync(string id) + { + var stopwatch = Stopwatch.StartNew(); + using (var entitiesContext = await _entitiesContextFactory.CreateAsync(readOnly: true)) + { + _logger.LogInformation("Fetching owners for package registration with ID {PackageId}.", id); + var owners = await entitiesContext + .PackageRegistrations + .Where(pr => pr.Id == id) + .Select(pr => pr.Owners.Select(u => u.Username).ToList()) + .FirstOrDefaultAsync(); + + if (owners == null) + { + _logger.LogWarning("No package registration with ID {PackageId} was found. Assuming no owners.", id); + return Array.Empty(); + } + + if (owners.Count == 0) + { + _logger.LogInformation("The package registration with ID {PackageId} has no owners.", id); + return Array.Empty(); + } + + // Sort the usernames in a consistent manner. + var sortedOwners = owners + .OrderBy(o => o, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + stopwatch.Stop(); + _telemetryService.TrackGetOwnersForPackageId(sortedOwners.Length, stopwatch.Elapsed); + _logger.LogInformation("The package registration with ID {PackageId} has {Count} owners.", id, sortedOwners.Length); + return sortedOwners; + } + } + + public async Task> GetVerifiedPackagesAsync() + { + return await RetrySqlAsync(async () => + { + var stopwatch = Stopwatch.StartNew(); + using (var connection = await _connectionFactory.OpenAsync()) + using (var command = connection.CreateCommand()) + { + command.CommandText = GetVerifiedPackagesSql; + command.CommandTimeout = SqlCommandTimeoutSeconds; + + using (var reader = await command.ExecuteReaderAsync()) + { + var output = new HashSet(StringComparer.OrdinalIgnoreCase); + while (await reader.ReadAsync()) + { + var id = reader.GetString(0); + output.Add(id); + } + + stopwatch.Stop(); + _telemetryService.TrackReadLatestVerifiedPackagesFromDatabase(output.Count, stopwatch.Elapsed); + + return output; + } + } + }); + } + + public async Task>> GetPackageIdToOwnersAsync() + { + return await RetrySqlAsync(async () => + { + var stopwatch = Stopwatch.StartNew(); + using (var connection = await _connectionFactory.OpenAsync()) + using (var command = connection.CreateCommand()) + { + command.CommandText = GetPackageIdToOwnersSql; + command.CommandTimeout = SqlCommandTimeoutSeconds; + + using (var reader = await command.ExecuteReaderAsync()) + { + var builder = new PackageIdToOwnersBuilder(_logger); + while (await reader.ReadAsync()) + { + var id = reader.GetString(0); + var username = reader.GetString(1); + + builder.Add(id, username); + } + + var output = builder.GetResult(); + stopwatch.Stop(); + _telemetryService.TrackReadLatestOwnersFromDatabase(output.Count, stopwatch.Elapsed); + + return output; + } + } + }); + } + + public async Task GetPopularityTransfersAsync() + { + return await RetrySqlAsync(async () => + { + var stopwatch = Stopwatch.StartNew(); + var output = new PopularityTransferData(); + using (var connection = await _connectionFactory.OpenAsync()) + using (var command = connection.CreateCommand()) + { + command.CommandText = GetPopularityTransfersSql; + command.CommandTimeout = SqlCommandTimeoutSeconds; + command.Parameters.Add(GetPopularityTransfersSkipParameter, SqlDbType.Int); + command.Parameters.AddWithValue(GetPopularityTransfersTakeParameter, GetPopularityTransfersPageSize); + + // Load popularity transfers by paging through the database. + // We continue paging until we receive fewer results than the page size. + int currentPageResults; + int totalResults = 0; + do + { + command.Parameters[GetPopularityTransfersSkipParameter].Value = totalResults; + + using (var reader = await command.ExecuteReaderAsync()) + { + currentPageResults = 0; + + while (await reader.ReadAsync()) + { + currentPageResults++; + + var fromId = reader.GetString(0); + var toId = reader.GetString(1); + + output.AddTransfer(fromId, toId); + } + } + + totalResults += currentPageResults; + } + while (currentPageResults == GetPopularityTransfersPageSize); + + stopwatch.Stop(); + _telemetryService.TrackReadLatestPopularityTransfersFromDatabase(output.Count, stopwatch.Elapsed); + + return output; + } + }); + } + + private async Task RetrySqlAsync(Func> actAsync) + { + var output = default(T); + await Retry.IncrementalAsync( + async () => + { + output = await actAsync(); + }, + ex => ex is SqlException, + maxRetries: 5, + initialWaitInterval: TimeSpan.Zero, + waitIncrement: TimeSpan.FromSeconds(10)); + return output; + } + } +} + diff --git a/src/NuGet.Services.AzureSearch/Db2AzureSearch/Db2AzureSearchCommand.cs b/src/NuGet.Services.AzureSearch/Db2AzureSearch/Db2AzureSearchCommand.cs new file mode 100644 index 000000000..5be9af6a5 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Db2AzureSearch/Db2AzureSearchCommand.cs @@ -0,0 +1,272 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Protocol.Catalog; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.AzureSearch.Catalog2AzureSearch; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.Db2AzureSearch +{ + public class Db2AzureSearchCommand : IAzureSearchCommand + { + private readonly INewPackageRegistrationProducer _producer; + private readonly IPackageEntityIndexActionBuilder _indexActionBuilder; + private readonly IBlobContainerBuilder _blobContainerBuilder; + private readonly IIndexBuilder _indexBuilder; + private readonly Func _batchPusherFactory; + private readonly ICatalogClient _catalogClient; + private readonly IStorageFactory _storageFactory; + private readonly IOwnerDataClient _ownerDataClient; + private readonly IDownloadDataClient _downloadDataClient; + private readonly IVerifiedPackagesDataClient _verifiedPackagesDataClient; + private readonly IPopularityTransferDataClient _popularityTransferDataClient; + private readonly IOptionsSnapshot _options; + private readonly IOptionsSnapshot _developmentOptions; + private readonly ILogger _logger; + + public Db2AzureSearchCommand( + INewPackageRegistrationProducer producer, + IPackageEntityIndexActionBuilder indexActionBuilder, + IBlobContainerBuilder blobContainerBuilder, + IIndexBuilder indexBuilder, + Func batchPusherFactory, + ICatalogClient catalogClient, + IStorageFactory storageFactory, + IOwnerDataClient ownerDataClient, + IDownloadDataClient downloadDataClient, + IVerifiedPackagesDataClient verifiedPackagesDataClient, + IPopularityTransferDataClient popularityTransferDataClient, + IOptionsSnapshot options, + IOptionsSnapshot developmentOptions, + ILogger logger) + { + _producer = producer ?? throw new ArgumentNullException(nameof(producer)); + _indexActionBuilder = indexActionBuilder ?? throw new ArgumentNullException(nameof(indexActionBuilder)); + _blobContainerBuilder = blobContainerBuilder ?? throw new ArgumentNullException(nameof(blobContainerBuilder)); + _indexBuilder = indexBuilder ?? throw new ArgumentNullException(nameof(indexBuilder)); + _batchPusherFactory = batchPusherFactory ?? throw new ArgumentNullException(nameof(batchPusherFactory)); + _catalogClient = catalogClient ?? throw new ArgumentNullException(nameof(catalogClient)); + _storageFactory = storageFactory ?? throw new ArgumentNullException(nameof(storageFactory)); + _ownerDataClient = ownerDataClient ?? throw new ArgumentNullException(nameof(ownerDataClient)); + _downloadDataClient = downloadDataClient ?? throw new ArgumentNullException(nameof(downloadDataClient)); + _verifiedPackagesDataClient = verifiedPackagesDataClient ?? throw new ArgumentNullException(nameof(verifiedPackagesDataClient)); + _popularityTransferDataClient = popularityTransferDataClient ?? throw new ArgumentNullException(nameof(popularityTransferDataClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _developmentOptions = developmentOptions ?? throw new ArgumentNullException(nameof(developmentOptions)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (_options.Value.MaxConcurrentBatches <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(options), + $"The {nameof(AzureSearchJobConfiguration.MaxConcurrentBatches)} must be greater than zero."); + } + } + + public async Task ExecuteAsync() + { + await ExecuteAsync(CancellationToken.None); + } + + private async Task ExecuteAsync(CancellationToken token) + { + using (var cancelledCts = new CancellationTokenSource()) + using (var produceWorkCts = new CancellationTokenSource()) + { + // Initialize the indexes, container and excluded packages data. + await InitializeAsync(); + + // Here, we fetch the current catalog timestamp to use as the initial cursor value for + // catalog2azuresearch. The idea here is that database is always more up-to-date than the catalog. + // We're about to read the database so if we capture a catalog timestamp now, we are guaranteed that + // any data we get from a database query will be more recent than the data represented by this catalog + // timestamp. When catalog2azuresearch starts up for the first time to update the index produced by this + // job, it will probably encounter some duplicate packages, but this is okay. + // + // Note that we could capture any dependency cursors here instead of catalog cursor, but this is + // pointless because there is no reliable way to filter out data fetched from the database based on a + // catalog-based cursor value. Suppose the dependency cursor is catalog2registration. If + // catalog2registration is very behind, then the index produced by this job will include packages that + // are not yet restorable (since they are not in the registration hives). This could lead to a case + // where a user is able to search for a package that he cannot restore. We mitigate this risk by + // trusting that our end-to-end tests will fail when catalog2registration (or any other V3 component) is + // broken, this blocking the deployment of new Azure Search indexes. + var catalogIndex = await _catalogClient.GetIndexAsync(_options.Value.CatalogIndexUrl); + var initialCursorValue = catalogIndex.CommitTimestamp; + _logger.LogInformation("The initial cursor value will be {CursorValue:O}.", initialCursorValue); + + var initialAuxiliaryData = await PushAllPackageRegistrationsAsync(cancelledCts, produceWorkCts); + + // Write the owner data file. + await WriteOwnerDataAsync(initialAuxiliaryData.Owners); + + // Write the download data file. + await WriteDownloadDataAsync(initialAuxiliaryData.Downloads); + + // Write the verified packages data file. + await WriteVerifiedPackagesDataAsync(initialAuxiliaryData.VerifiedPackages); + + // Write popularity transfers data file. + await WritePopularityTransfersDataAsync(initialAuxiliaryData.PopularityTransfers); + + // Write the cursor. + _logger.LogInformation("Writing the initial cursor value to be {CursorValue:O}.", initialCursorValue); + var frontCursorStorage = _storageFactory.Create(); + var frontCursor = new DurableCursor( + frontCursorStorage.ResolveUri(Catalog2AzureSearchCommand.CursorRelativeUri), + frontCursorStorage, + DateTime.MinValue); + frontCursor.Value = initialCursorValue.UtcDateTime; + await frontCursor.SaveAsync(token); + } + } + + private async Task InitializeAsync() + { + var containerDeleted = false; + if (_developmentOptions.Value.ReplaceContainersAndIndexes) + { + containerDeleted = await _blobContainerBuilder.DeleteIfExistsAsync(); + await _indexBuilder.DeleteSearchIndexIfExistsAsync(); + await _indexBuilder.DeleteHijackIndexIfExistsAsync(); + } + + await _blobContainerBuilder.CreateAsync(containerDeleted); + await _indexBuilder.CreateSearchIndexAsync(); + await _indexBuilder.CreateHijackIndexAsync(); + } + + private async Task PushAllPackageRegistrationsAsync( + CancellationTokenSource cancelledCts, + CancellationTokenSource produceWorkCts) + { + _logger.LogInformation("Pushing all packages to Azure Search and initializing version lists."); + var allWork = new ConcurrentBag(); + var producerTask = ProduceWorkAsync(allWork, produceWorkCts, cancelledCts.Token); + var consumerTasks = Enumerable + .Range(0, _options.Value.MaxConcurrentBatches) + .Select(i => ConsumeWorkAsync(allWork, produceWorkCts.Token, cancelledCts.Token)) + .ToList(); + var allTasks = new[] { producerTask }.Concat(consumerTasks).ToList(); + + // If one of the tasks throws an exception before the work is completed, cancel the work. + var firstTask = await Task.WhenAny(allTasks); + if (firstTask.IsFaulted) + { + cancelledCts.Cancel(); + } + + await firstTask; + await Task.WhenAll(allTasks); + _logger.LogInformation("Done initializing the Azure Search indexes and version lists."); + + return await producerTask; + } + + private async Task WriteOwnerDataAsync(SortedDictionary> owners) + { + _logger.LogInformation("Writing the initial owners file."); + await _ownerDataClient.ReplaceLatestIndexedAsync( + owners, + AccessConditionWrapper.GenerateIfNotExistsCondition()); + _logger.LogInformation("Done uploading the initial owners file."); + } + + private async Task WriteDownloadDataAsync(DownloadData downloadData) + { + _logger.LogInformation("Writing the initial download data file."); + await _downloadDataClient.ReplaceLatestIndexedAsync( + downloadData, + AccessConditionWrapper.GenerateIfNotExistsCondition()); + _logger.LogInformation("Done uploading the initial download data file."); + } + + private async Task WriteVerifiedPackagesDataAsync(HashSet verifiedPackages) + { + _logger.LogInformation("Writing the initial verified packages data file."); + await _verifiedPackagesDataClient.ReplaceLatestAsync( + verifiedPackages, + AccessConditionWrapper.GenerateIfNotExistsCondition()); + _logger.LogInformation("Done uploading the initial verified packages data file."); + } + + private async Task WritePopularityTransfersDataAsync(PopularityTransferData popularityTransfers) + { + _logger.LogInformation("Writing the initial popularity transfers data file."); + await _popularityTransferDataClient.ReplaceLatestIndexedAsync( + popularityTransfers, + AccessConditionWrapper.GenerateIfNotExistsCondition()); + _logger.LogInformation("Done uploading the initial popularity transfers data file."); + } + + private async Task ProduceWorkAsync( + ConcurrentBag allWork, + CancellationTokenSource produceWorkCts, + CancellationToken cancellationToken) + { + await Task.Yield(); + var output = await _producer.ProduceWorkAsync(allWork, cancellationToken); + produceWorkCts.Cancel(); + return output; + } + + private async Task ConsumeWorkAsync( + ConcurrentBag allWork, + CancellationToken produceWorkToken, + CancellationToken cancellationToken) + { + await Task.Yield(); + + var batchPusher = _batchPusherFactory(); + + NewPackageRegistration work = null; + try + { + while ((allWork.TryTake(out work) || !produceWorkToken.IsCancellationRequested) + && !cancellationToken.IsCancellationRequested) + { + // If there's no work to do, wait a bit before checking again. + if (work == null) + { + await Task.Delay(TimeSpan.FromMilliseconds(100)); + continue; + } + + var indexActions = _indexActionBuilder.AddNewPackageRegistration(work); + + // There can be an empty set of index actions if there were no packages associated with this + // package registration. + if (!indexActions.IsEmpty) + { + batchPusher.EnqueueIndexActions(work.PackageId, indexActions); + var fullBatchesResult = await batchPusher.TryPushFullBatchesAsync(); + fullBatchesResult.EnsureSuccess(); + } + } + + var finishResult = await batchPusher.TryFinishAsync(); + finishResult.EnsureSuccess(); + } + catch (Exception ex) + { + _logger.LogError( + 0, + ex, + "An exception was thrown while processing package ID {PackageId}.", + work?.PackageId ?? "(last batch...)"); + throw; + } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Db2AzureSearch/Db2AzureSearchConfiguration.cs b/src/NuGet.Services.AzureSearch/Db2AzureSearch/Db2AzureSearchConfiguration.cs new file mode 100644 index 000000000..acf9e08c9 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Db2AzureSearch/Db2AzureSearchConfiguration.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.AzureSearch.AuxiliaryFiles; + +namespace NuGet.Services.AzureSearch.Db2AzureSearch +{ + public class Db2AzureSearchConfiguration : AzureSearchJobConfiguration, IAuxiliaryDataStorageConfiguration + { + public int DatabaseBatchSize { get; set; } = 10000; + public string CatalogIndexUrl { get; set; } + public string AuxiliaryDataStorageConnectionString { get; set; } + public string AuxiliaryDataStorageContainer { get; set; } + public string AuxiliaryDataStorageDownloadsPath { get; set; } + public string AuxiliaryDataStorageExcludedPackagesPath { get; set; } + public string AuxiliaryDataStorageVerifiedPackagesPath { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Db2AzureSearch/Db2AzureSearchDevelopmentConfiguration.cs b/src/NuGet.Services.AzureSearch/Db2AzureSearch/Db2AzureSearchDevelopmentConfiguration.cs new file mode 100644 index 000000000..a2a6bfef5 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Db2AzureSearch/Db2AzureSearchDevelopmentConfiguration.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; + +namespace NuGet.Services.AzureSearch.Db2AzureSearch +{ + /// + /// Settings used at development time that should not be used in production environments. + /// + public class Db2AzureSearchDevelopmentConfiguration : AzureSearchJobDevelopmentConfiguration + { + /// + /// If true, deletes the existing Azure Storage containers and Azure Search indexes. + /// This should be false on production environments. + /// + public bool ReplaceContainersAndIndexes { get; set; } + + /// + /// Db2AzureSearch skips packages whose ID start with these prefixes. + /// This is case insensitive. This should be empty on production environments. + /// + public IReadOnlyList SkipPackagePrefixes { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Db2AzureSearch/EnumerableExtensions.cs b/src/NuGet.Services.AzureSearch/Db2AzureSearch/EnumerableExtensions.cs new file mode 100644 index 000000000..e7008e502 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Db2AzureSearch/EnumerableExtensions.cs @@ -0,0 +1,64 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; + +namespace NuGet.Services.AzureSearch.Db2AzureSearch +{ + internal static class EnumerableExtensions + { + public static IEnumerable> Batch( + this IEnumerable sequence, + Func getItemSize, + int desiredSize) + { + if (sequence == null) + { + throw new ArgumentNullException(nameof(sequence)); + } + + if (getItemSize == null) + { + throw new ArgumentNullException(nameof(getItemSize)); + } + + if (desiredSize < 0) + { + throw new ArgumentOutOfRangeException( + nameof(desiredSize), + "The max size must be greater than or equal to zero."); + } + + var list = new List(); + var sizeSoFar = 0; + + foreach (var item in sequence) + { + var itemSize = getItemSize(item); + + if (sizeSoFar + itemSize > desiredSize) + { + if (list.Count > 0) + { + yield return list; + } + + list = new List { item }; + sizeSoFar = 0; + } + else + { + list.Add(item); + } + + sizeSoFar += itemSize; + } + + if (list.Count > 0) + { + yield return list; + } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Db2AzureSearch/INewPackageRegistrationProducer.cs b/src/NuGet.Services.AzureSearch/Db2AzureSearch/INewPackageRegistrationProducer.cs new file mode 100644 index 000000000..4418bd4d8 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Db2AzureSearch/INewPackageRegistrationProducer.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Concurrent; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch.Db2AzureSearch +{ + public interface INewPackageRegistrationProducer + { + Task ProduceWorkAsync( + ConcurrentBag allWork, + CancellationToken cancellationToken); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Db2AzureSearch/IPackageEntityIndexActionBuilder.cs b/src/NuGet.Services.AzureSearch/Db2AzureSearch/IPackageEntityIndexActionBuilder.cs new file mode 100644 index 000000000..72d09dbe0 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Db2AzureSearch/IPackageEntityIndexActionBuilder.cs @@ -0,0 +1,10 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.Db2AzureSearch +{ + public interface IPackageEntityIndexActionBuilder + { + IndexActions AddNewPackageRegistration(NewPackageRegistration packageRegistration); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Db2AzureSearch/InitialAuxiliaryData.cs b/src/NuGet.Services.AzureSearch/Db2AzureSearch/InitialAuxiliaryData.cs new file mode 100644 index 000000000..5938960aa --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Db2AzureSearch/InitialAuxiliaryData.cs @@ -0,0 +1,32 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Services.AzureSearch.AuxiliaryFiles; + +namespace NuGet.Services.AzureSearch.Db2AzureSearch +{ + public class InitialAuxiliaryData + { + public InitialAuxiliaryData( + SortedDictionary> owners, + DownloadData downloads, + HashSet excludedPackages, + HashSet verifiedPackages, + PopularityTransferData popularityTransfers) + { + Owners = owners ?? throw new ArgumentNullException(nameof(owners)); + Downloads = downloads ?? throw new ArgumentNullException(nameof(downloads)); + ExcludedPackages = excludedPackages ?? throw new ArgumentNullException(nameof(excludedPackages)); + VerifiedPackages = verifiedPackages ?? throw new ArgumentNullException(nameof(verifiedPackages)); + PopularityTransfers = popularityTransfers ?? throw new ArgumentNullException(nameof(popularityTransfers)); + } + + public SortedDictionary> Owners { get; } + public DownloadData Downloads { get; } + public HashSet ExcludedPackages { get; } + public HashSet VerifiedPackages { get; } + public PopularityTransferData PopularityTransfers { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Db2AzureSearch/NewPackageRegistration.cs b/src/NuGet.Services.AzureSearch/Db2AzureSearch/NewPackageRegistration.cs new file mode 100644 index 000000000..10380be3a --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Db2AzureSearch/NewPackageRegistration.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Services.Entities; + +namespace NuGet.Services.AzureSearch.Db2AzureSearch +{ + /// + /// The information required to bring an entire package registration up to date in the Azure Search indexes. This + /// data is populated from the database and storage by db2azuresearch. + /// + public class NewPackageRegistration + { + public NewPackageRegistration( + string packageId, + long totalDownloadCount, + string[] owners, + IReadOnlyList packages, + bool isExcludedByDefault) + { + PackageId = packageId ?? throw new ArgumentNullException(packageId); + TotalDownloadCount = totalDownloadCount; + Owners = owners ?? throw new ArgumentNullException(nameof(owners)); + Packages = packages ?? throw new ArgumentNullException(nameof(packages)); + IsExcludedByDefault = isExcludedByDefault; + } + + public string PackageId { get; } + public long TotalDownloadCount { get; } + public string[] Owners { get; } + public IReadOnlyList Packages { get; } + public bool IsExcludedByDefault { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Db2AzureSearch/NewPackageRegistrationProducer.cs b/src/NuGet.Services.AzureSearch/Db2AzureSearch/NewPackageRegistrationProducer.cs new file mode 100644 index 000000000..11ba3c54a --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Db2AzureSearch/NewPackageRegistrationProducer.cs @@ -0,0 +1,411 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Data.Entity; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.Entities; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.Db2AzureSearch +{ + public class NewPackageRegistrationProducer : INewPackageRegistrationProducer + { + private readonly IEntitiesContextFactory _contextFactory; + private readonly IAuxiliaryFileClient _auxiliaryFileClient; + private readonly IDatabaseAuxiliaryDataFetcher _databaseFetcher; + private readonly IDownloadTransferrer _downloadTransferrer; + private readonly IFeatureFlagService _featureFlags; + private readonly IOptionsSnapshot _options; + private readonly IOptionsSnapshot _developmentOptions; + private readonly ILogger _logger; + + public NewPackageRegistrationProducer( + IEntitiesContextFactory contextFactory, + IAuxiliaryFileClient auxiliaryFileClient, + IDatabaseAuxiliaryDataFetcher databaseFetcher, + IDownloadTransferrer downloadTransferrer, + IFeatureFlagService featureFlags, + IOptionsSnapshot options, + IOptionsSnapshot developmentOptions, + ILogger logger) + { + _contextFactory = contextFactory ?? throw new ArgumentNullException(nameof(contextFactory)); + _auxiliaryFileClient = auxiliaryFileClient ?? throw new ArgumentNullException(nameof(auxiliaryFileClient)); + _databaseFetcher = databaseFetcher ?? throw new ArgumentNullException(nameof(databaseFetcher)); + _downloadTransferrer = downloadTransferrer ?? throw new ArgumentNullException(nameof(downloadTransferrer)); + _featureFlags = featureFlags ?? throw new ArgumentNullException(nameof(featureFlags)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _developmentOptions = developmentOptions ?? throw new ArgumentNullException(nameof(developmentOptions)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ProduceWorkAsync( + ConcurrentBag allWork, + CancellationToken cancellationToken) + { + var ranges = await GetPackageRegistrationRangesAsync(); + + // Fetch exclude packages list from auxiliary files. + // These packages are excluded from the default search's results. + var excludedPackages = await _auxiliaryFileClient.LoadExcludedPackagesAsync(); + + Guard.Assert( + excludedPackages.Comparer == StringComparer.OrdinalIgnoreCase, + $"Excluded packages HashSet should be using {nameof(StringComparer.OrdinalIgnoreCase)}"); + + // Fetch the download data from the auxiliary file, since this is what is used for displaying download + // counts in the search service. We don't use the gallery DB values as they are different from the + // auxiliary file. + var downloads = await _auxiliaryFileClient.LoadDownloadDataAsync(); + var popularityTransfers = await GetPopularityTransfersAsync(); + + // Apply changes from popularity transfers. + var transferredDownloads = GetTransferredDownloads(downloads, popularityTransfers); + + // Build a list of the owners data and verified IDs as we collect package registrations from the database. + var ownersBuilder = new PackageIdToOwnersBuilder(_logger); + var verifiedPackages = new HashSet(StringComparer.OrdinalIgnoreCase); + + for (var i = 0; i < ranges.Count && !cancellationToken.IsCancellationRequested; i++) + { + if (ShouldWait(allWork, log: true)) + { + while (ShouldWait(allWork, log: false)) + { + await Task.Delay(TimeSpan.FromSeconds(1)); + } + + _logger.LogInformation("Resuming fetching package registrations from the database."); + } + + var range = ranges[i]; + + var allPackages = await GetPackagesAsync(range); + var keyToPackages = allPackages + .GroupBy(x => x.PackageRegistrationKey) + .ToDictionary(x => x.Key, x => x.ToList()); + + var packageRegistrationInfo = await GetPackageRegistrationInfoAsync(range); + + foreach (var pr in packageRegistrationInfo) + { + if (!transferredDownloads.TryGetValue(pr.Id, out var packageDownloads)) + { + packageDownloads = 0; + } + + if (!keyToPackages.TryGetValue(pr.Key, out var packages)) + { + packages = new List(); + } + + var isExcludedByDefault = excludedPackages.Contains(pr.Id); + + allWork.Add(new NewPackageRegistration( + pr.Id, + packageDownloads, + pr.Owners, + packages, + isExcludedByDefault)); + + ownersBuilder.Add(pr.Id, pr.Owners); + + if (pr.IsVerified) + { + verifiedPackages.Add(pr.Id); + } + } + + _logger.LogInformation("Done initializing batch {Number}/{Count}.", i + 1, ranges.Count); + } + + return new InitialAuxiliaryData( + ownersBuilder.GetResult(), + downloads, + excludedPackages, + verifiedPackages, + popularityTransfers); + } + + private bool ShouldWait(ConcurrentBag allWork, bool log) + { + var packageCount = allWork.Sum(x => x.Packages.Count); + var max = 2 * _options.Value.DatabaseBatchSize; + + if (packageCount > max) + { + if (log) + { + _logger.LogInformation( + "There are {PackageCount} packages in memory waiting to be pushed to Azure Search. " + + "Waiting until this number drops below {Max} before fetching more packages.", + packageCount, + max); + } + + return true; + } + + return false; + } + + private async Task GetPopularityTransfersAsync() + { + if (!_options.Value.EnablePopularityTransfers) + { + _logger.LogWarning( + "Popularity transfers are disabled. Popularity transfers will be ignored."); + return new PopularityTransferData(); + } + + if (!_featureFlags.IsPopularityTransferEnabled()) + { + _logger.LogWarning( + "Popularity transfers feature flag is disabled. " + + "Popularity transfers will be ignored."); + return new PopularityTransferData(); + } + + return await _databaseFetcher.GetPopularityTransfersAsync(); + } + + private Dictionary GetTransferredDownloads( + DownloadData downloads, + PopularityTransferData popularityTransfers) + { + var transferChanges = _downloadTransferrer.InitializeDownloadTransfers( + downloads, + popularityTransfers); + + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var packageDownload in downloads) + { + result[packageDownload.Key] = packageDownload.Value.Total; + } + + foreach (var transferChange in transferChanges) + { + result[transferChange.Key] = transferChange.Value; + } + + return result; + } + + private async Task> GetPackagesAsync(PackageRegistrationRange range) + { + using (var context = await CreateContextAsync()) + { + var minKey = range.MinKey; + var query = context + .Set() + .Include(x => x.PackageRegistration) + .Include(x => x.PackageTypes) + .Where(p => p.PackageStatusKey == PackageStatus.Available) + .Where(p => p.PackageRegistrationKey >= minKey); + + if (range.MaxKey.HasValue) + { + var maxKey = range.MaxKey.Value; + query = query + .Where(p => p.PackageRegistrationKey <= maxKey); + } + + LogFetching("packages", range); + + return await query.ToListAsync(); + } + } + + private void LogFetching(string fetched, PackageRegistrationRange range) + { + if (range.MaxKey.HasValue) + { + _logger.LogInformation( + "Fetching " + fetched + " with package registration key >= {MinKey} and <= {MaxKey} (~{Count} packages).", + range.MinKey, + range.MaxKey, + range.PackageCount); + } + else + { + _logger.LogInformation("Fetching " + fetched + " with package registration key >= {MinKey} (~{Count} packages).", + range.MinKey, + range.PackageCount); + } + } + + private async Task> GetPackageRegistrationInfoAsync(PackageRegistrationRange range) + { + using (var context = await CreateContextAsync()) + { + var minKey = range.MinKey; + var query = context + .Set() + .Include(x => x.Owners) + .Where(pr => pr.Key >= minKey); + + if (range.MaxKey.HasValue) + { + var maxKey = range.MaxKey.Value; + query = query + .Where(pr => pr.Key <= maxKey); + } + + LogFetching("owners", range); + + var packageRegistrations = await query.ToListAsync(); + + return packageRegistrations + .Where(pr => !ShouldSkipPackageRegistration(pr)) + .Select(pr => new PackageRegistrationInfo( + pr.Key, + pr.Id, + pr.Owners.Select(x => x.Username).ToArray(), + pr.IsVerified)) + .ToList(); + } + } + + private bool ShouldSkipPackageRegistration(PackageRegistration packageRegistration) + { + // Capture the skip list to avoid reload issues. + var skipPrefixes = _developmentOptions.Value.SkipPackagePrefixes; + if (skipPrefixes == null) + { + return false; + } + + foreach (var skipPrefix in skipPrefixes) + { + if (packageRegistration.Id.StartsWith(skipPrefix, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + private async Task> GetPackageRegistrationRangesAsync() + { + using (var context = await CreateContextAsync()) + { + _logger.LogInformation("Fetching all package registration keys and their available package counts."); + + // Get the number of packages per package registration key, in ascending order. + var stopwatch = Stopwatch.StartNew(); + var packageCounts = await context + .Set() + .OrderBy(pr => pr.Key) + .Select(pr => new + { + pr.Key, + PackageCount = pr.Packages.Where(p => p.PackageStatusKey == PackageStatus.Available).Count() + }) + .ToListAsync(); + var totalPackages = packageCounts.Sum(pr => pr.PackageCount); + + // Sequentially group the package registrations up to a maximum batch size. If a single package + // registration has a package count that is more than the batch size, it will be in its own batch. + var batches = packageCounts + .Batch(pr => pr.PackageCount, _options.Value.DatabaseBatchSize) + .ToList(); + + _logger.LogInformation( + "Got {Count} package registrations, {BatchCount} batches, which have {PackageCount} packages. Took {Duration}.", + packageCounts.Count, + batches.Count, + totalPackages, + stopwatch.Elapsed); + + // For each batch, generate a package registration key range. These range of keys collectively cover all + // possible integer keys. We want to cover all possible integer keys so that we very clearly will avoid + // missing any data. + var ranges = new List(); + for (var i = 0; i < batches.Count; i++) + { + int minKey; + if (i == 0) + { + minKey = 1; + } + else + { + minKey = batches[i][0].Key; + } + + int? maxKey; + if (i < batches.Count - 1) + { + maxKey = batches[i + 1][0].Key - 1; + } + else + { + maxKey = null; + } + + var packageCount = batches[i].Sum(x => x.PackageCount); + ranges.Add(new PackageRegistrationRange(minKey, maxKey, packageCount)); + } + + return ranges; + } + } + + private async Task CreateContextAsync() + { + return await _contextFactory.CreateAsync(readOnly: true); + } + + private class PackageRegistrationRange + { + public PackageRegistrationRange(int minKey, int? maxKey, int packageCount) + { + MinKey = minKey; + MaxKey = maxKey; + PackageCount = packageCount; + } + + /// + /// Inclusive package registration key minimum. + /// + public int MinKey { get; } + + /// + /// Inclusive package registration key maximum. If this value is null, the range has an unbounded maximum. + /// + public int? MaxKey { get; } + + /// + /// The estimated number of packages in this range. + /// + public int PackageCount { get; } + } + + private class PackageRegistrationInfo + { + public PackageRegistrationInfo(int key, string id, string[] owners, bool isVerified) + { + Key = key; + Id = id; + Owners = owners; + IsVerified = isVerified; + } + + public int Key { get; } + public string Id { get; } + public string[] Owners { get; } + public bool IsVerified { get; } + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Db2AzureSearch/PackageEntityIndexActionBuilder.cs b/src/NuGet.Services.AzureSearch/Db2AzureSearch/PackageEntityIndexActionBuilder.cs new file mode 100644 index 000000000..5e796f39b --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Db2AzureSearch/PackageEntityIndexActionBuilder.cs @@ -0,0 +1,177 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Logging; +using NuGet.Services.Entities; +using NuGet.Versioning; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.Db2AzureSearch +{ + public class PackageEntityIndexActionBuilder : IPackageEntityIndexActionBuilder + { + private readonly ISearchDocumentBuilder _search; + private readonly IHijackDocumentBuilder _hijack; + private readonly ILogger _logger; + + public PackageEntityIndexActionBuilder( + ISearchDocumentBuilder search, + IHijackDocumentBuilder hijack, + ILogger logger) + { + _search = search ?? throw new ArgumentNullException(nameof(search)); + _hijack = hijack ?? throw new ArgumentNullException(nameof(hijack)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public IndexActions AddNewPackageRegistration(NewPackageRegistration packageRegistration) + { + var versionProperties = new Dictionary(); + var versionListData = new VersionListData(versionProperties); + var versionLists = new VersionLists(versionListData); + + var changes = packageRegistration + .Packages + .Select(GetVersionListChange) + .ToList(); + var indexChanges = versionLists.ApplyChanges(changes); + + var versionToPackage = packageRegistration + .Packages + .ToDictionary(p => NuGetVersion.Parse(p.Version)); + + var search = indexChanges + .Search + .Select(p => GetSearchIndexAction( + packageRegistration, + versionToPackage, + versionLists, + p.Key, + p.Value)) + .ToList(); + + var hijack = indexChanges + .Hijack + .Select(p => GetHijackIndexAction( + packageRegistration.PackageId, + versionToPackage[p.Key], + p.Value)) + .ToList(); + + return new IndexActions( + search, + hijack, + new ResultAndAccessCondition( + versionLists.GetVersionListData(), + AccessConditionWrapper.GenerateEmptyCondition())); + } + + private void VerifyConsistency( + string packageId, + Package package) + { + var parsedVersion = NuGetVersion.Parse(package.Version); + var normalizedString = parsedVersion.ToNormalizedString(); + + if (package.NormalizedVersion == null + || package.NormalizedVersion != parsedVersion.ToNormalizedString()) + { + var message = $"The calculated {nameof(Package.NormalizedVersion)} does not match the DB value."; + _logger.LogError( + message + ". ID: {PackageId}, DB: {DbValue}, Calculated: {CalculatedValue}", + packageId, + package.NormalizedVersion, + normalizedString); + throw new InvalidOperationException(message); + } + + if (package.IsPrerelease != parsedVersion.IsPrerelease) + { + var message = $"The calculated {nameof(Package.IsPrerelease)} does not match the DB value."; + _logger.LogError( + message + ". ID: {PackageId}, DB: {DbValue}, Calculated: {CalculatedValue}", + packageId, + package.IsPrerelease, + parsedVersion.IsPrerelease); + throw new InvalidOperationException(message); + } + } + + private static VersionListChange GetVersionListChange(Package x) + { + return VersionListChange.Upsert( + fullOrOriginalVersion: x.Version, + data: new VersionPropertiesData( + listed: x.Listed, + semVer2: x.SemVerLevelKey.HasValue && x.SemVerLevelKey.Value >= SemVerLevelKey.SemVer2)); + } + + private IndexAction GetSearchIndexAction( + NewPackageRegistration packageRegistration, + IReadOnlyDictionary versionToPackage, + VersionLists versionLists, + SearchFilters searchFilters, + SearchIndexChangeType changeType) + { + if (changeType == SearchIndexChangeType.Delete) + { + return IndexAction.Delete(_search.Keyed( + packageRegistration.PackageId, + searchFilters)); + } + + if (changeType != SearchIndexChangeType.AddFirst) + { + throw new ArgumentException( + $"The only change types supported are {nameof(SearchIndexChangeType.AddFirst)} and " + + $"{nameof(SearchIndexChangeType.Delete)}.", + nameof(changeType)); + } + + var latestFlags = _search.LatestFlagsOrNull(versionLists, searchFilters); + var package = versionToPackage[latestFlags.LatestVersionInfo.ParsedVersion]; + var owners = packageRegistration + .Owners + .OrderBy(u => u, StringComparer.InvariantCultureIgnoreCase) + .ToArray(); + + VerifyConsistency(packageRegistration.PackageId, package); + + return IndexAction.Upload(_search.FullFromDb( + packageRegistration.PackageId, + searchFilters, + latestFlags.LatestVersionInfo.ListedFullVersions, + latestFlags.IsLatestStable, + latestFlags.IsLatest, + latestFlags.LatestVersionInfo.FullVersion, + package, + owners, + packageRegistration.TotalDownloadCount, + packageRegistration.IsExcludedByDefault)); + } + + private IndexAction GetHijackIndexAction( + string packageId, + Package package, + HijackDocumentChanges changes) + { + if (!changes.UpdateMetadata) + { + throw new ArgumentException( + "The hijack document changes must be set to update metadata.", + nameof(changes)); + } + + VerifyConsistency(packageId, package); + + return IndexAction.Upload(_hijack.FullFromDb( + packageId, + changes, + package)); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs b/src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs new file mode 100644 index 000000000..0338d236d --- /dev/null +++ b/src/NuGet.Services.AzureSearch/DependencyInjectionExtensions.cs @@ -0,0 +1,325 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net.Http; +using Autofac; +using Microsoft.Azure.Search; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Rest; +using Microsoft.Rest.TransientFaultHandling; +using Microsoft.WindowsAzure.Storage; +using NuGet.Protocol; +using NuGet.Protocol.Catalog; +using NuGet.Services.AzureSearch.Auxiliary2AzureSearch; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.AzureSearch.Catalog2AzureSearch; +using NuGet.Services.AzureSearch.Db2AzureSearch; +using NuGet.Services.AzureSearch.SearchService; +using NuGet.Services.AzureSearch.Wrappers; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Services.V3; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch +{ + public static class DependencyInjectionExtensions + { + public static ContainerBuilder AddAzureSearch(this ContainerBuilder containerBuilder) + { + containerBuilder.AddFeatureFlags(); + + /// Here, we register services that depend on an interface that there are multiple implementations. + + /// There are multiple implementations of . + RegisterIndexServices(containerBuilder, "SearchIndex", "HijackIndex"); + + /// There are multiple implementations of storage, in particular . + RegisterAzureSearchStorageServices(containerBuilder, "AzureSearchStorage"); + RegisterAuxiliaryDataStorageServices(containerBuilder, "AuxiliaryDataStorage"); + + return containerBuilder; + } + + private static void RegisterIndexServices(ContainerBuilder containerBuilder, string searchIndexKey, string hijackIndexKey) + { + containerBuilder + .Register(c => + { + var serviceClient = c.Resolve(); + var options = c.Resolve>(); + return serviceClient.Indexes.GetClient(options.Value.SearchIndexName); + }) + .SingleInstance() + .Keyed(searchIndexKey); + + containerBuilder + .Register(c => + { + var serviceClient = c.Resolve(); + var options = c.Resolve>(); + return serviceClient.Indexes.GetClient(options.Value.HijackIndexName); + }) + .SingleInstance() + .Keyed(hijackIndexKey); + + containerBuilder + .Register(c => new BatchPusher( + c.ResolveKeyed(searchIndexKey), + c.ResolveKeyed(hijackIndexKey), + c.Resolve(), + c.Resolve>(), + c.Resolve>(), + c.Resolve(), + c.Resolve>())); + + containerBuilder + .Register(c => new AzureSearchService( + c.Resolve(), + c.ResolveKeyed(searchIndexKey), + c.ResolveKeyed(hijackIndexKey), + c.Resolve(), + c.Resolve())); + + containerBuilder + .Register(c => new SearchStatusService( + c.ResolveKeyed(searchIndexKey), + c.ResolveKeyed(hijackIndexKey), + c.Resolve(), + c.Resolve(), + c.Resolve(), + c.Resolve>(), + c.Resolve(), + c.Resolve>())); + } + + private static void RegisterAzureSearchStorageServices(ContainerBuilder containerBuilder, string key) + { + containerBuilder + .Register(c => + { + var options = c.Resolve>(); + return new CloudBlobClientWrapper( + options.Value.StorageConnectionString, + DefaultBlobRequestOptions.Create()); + }) + .Keyed(key); + + containerBuilder + .Register(c => new VersionListDataClient( + c.ResolveKeyed(key), + c.Resolve>(), + c.Resolve>())); + + containerBuilder + .Register(c => + { + var options = c.Resolve>(); + return CloudStorageAccount.Parse(options.Value.StorageConnectionString); + }) + .Keyed(key); + + containerBuilder + .Register(c => + { + var options = c.Resolve>(); + return new AzureStorageFactory( + c.ResolveKeyed(key), + options.Value.StorageContainer, + maxExecutionTime: AzureStorage.DefaultMaxExecutionTime, + serverTimeout: AzureStorage.DefaultServerTimeout, + path: options.Value.NormalizeStoragePath(), + baseAddress: null, + useServerSideCopy: true, + compressContent: false, + verbose: true, + initializeContainer: false, + throttle: NullThrottle.Instance); + }) + .Keyed(key); + + containerBuilder + .Register(c => new BlobContainerBuilder( + c.ResolveKeyed(key), + c.Resolve>(), + c.Resolve>())); + + containerBuilder + .Register(c => new DownloadDataClient( + c.ResolveKeyed(key), + c.Resolve>(), + c.Resolve(), + c.Resolve>())); + + containerBuilder + .Register(c => new VerifiedPackagesDataClient( + c.ResolveKeyed(key), + c.Resolve>(), + c.Resolve(), + c.Resolve>())); + + containerBuilder + .Register(c => new OwnerDataClient( + c.ResolveKeyed(key), + c.Resolve>(), + c.Resolve(), + c.Resolve>())); + + containerBuilder + .Register(c => new PopularityTransferDataClient( + c.ResolveKeyed(key), + c.Resolve>(), + c.Resolve(), + c.Resolve>())); + + containerBuilder + .Register(c => new Catalog2AzureSearchCommand( + c.Resolve(), + c.ResolveKeyed(key), + c.Resolve>(), + c.Resolve(), + c.Resolve(), + c.Resolve>(), + c.Resolve>())); + + containerBuilder + .Register(c => new Db2AzureSearchCommand( + c.Resolve(), + c.Resolve(), + c.Resolve(), + c.Resolve(), + c.Resolve>(), + c.Resolve(), + c.ResolveKeyed(key), + c.Resolve(), + c.Resolve(), + c.Resolve(), + c.Resolve(), + c.Resolve>(), + c.Resolve>(), + c.Resolve>())); + } + + private static void RegisterAuxiliaryDataStorageServices(ContainerBuilder containerBuilder, string key) + { + containerBuilder + .Register(c => + { + var options = c.Resolve>(); + return new CloudBlobClientWrapper( + options.Value.AuxiliaryDataStorageConnectionString, + DefaultBlobRequestOptions.Create()); + }) + .Keyed(key); + + containerBuilder + .Register(c => new AuxiliaryFileClient( + c.ResolveKeyed(key), + c.Resolve>(), + c.Resolve(), + c.Resolve>())); + } + + public static IServiceCollection AddAzureSearch( + this IServiceCollection services, + IDictionary telemetryGlobalDimensions) + { + services.AddV3(telemetryGlobalDimensions); + + services.AddFeatureFlags(); + services.AddTransient(); + + services.AddTransient(p => new SearchServiceClientWrapper( + p.GetRequiredService(), + GetSearchDelegatingHandlers(p.GetRequiredService()), + GetSearchRetryPolicy(), + p.GetRequiredService>())); + + services + .AddTransient(p => + { + var options = p.GetRequiredService>(); + + var client = new SearchServiceClient( + options.Value.SearchServiceName, + new SearchCredentials(options.Value.SearchServiceApiKey), + new WebRequestHandler(), + GetSearchDelegatingHandlers(p.GetRequiredService())); + + client.SetRetryPolicy(GetSearchRetryPolicy()); + + return client; + }); + + services.AddSingleton(); + services.AddScoped(p => p.GetRequiredService().Get()); + services.AddSingleton(); + + services.AddSingleton(); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(p => new Auxiliary2AzureSearchCommand( + p.GetRequiredService(), + p.GetRequiredService(), + p.GetRequiredService(), + p.GetRequiredService(), + p.GetRequiredService>())); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + return services; + } + + /// + /// Defaults originally taken from: + /// https://github.com/Azure/azure-sdk-for-net/blob/96421089bc26198098f320ea50e0208e98376956/sdk/mgmtcommon/ClientRuntime/ClientRuntime/RetryDelegatingHandler.cs#L19-L22 + /// + /// Note that this policy only applied to the automatically initialized by + /// the Azure Search SDK. This policy does not apply to . + /// + private static RetryPolicy GetSearchRetryPolicy() + { + return new RetryPolicy( + new HttpStatusCodeErrorDetectionStrategy(), + retryCount: 3, + minBackoff: TimeSpan.FromSeconds(1), + maxBackoff: TimeSpan.FromSeconds(10), + deltaBackoff: TimeSpan.FromSeconds(10)); + } + + public static DelegatingHandler[] GetSearchDelegatingHandlers(ILoggerFactory loggerFactory) + { + return new[] + { + new WebExceptionRetryDelegatingHandler(loggerFactory.CreateLogger()), + }; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/DocumentUtilities.cs b/src/NuGet.Services.AzureSearch/DocumentUtilities.cs new file mode 100644 index 000000000..f3d7230cd --- /dev/null +++ b/src/NuGet.Services.AzureSearch/DocumentUtilities.cs @@ -0,0 +1,69 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Web; + +namespace NuGet.Services.AzureSearch +{ + public static class DocumentUtilities + { + public static readonly IReadOnlyList AllSearchFilters = Enum + .GetValues(typeof(SearchFilters)) + .Cast() + .ToList(); + + public static string GetSearchFilterString(SearchFilters searchFilters) + { + return searchFilters.ToString(); + } + + public static string GetSearchDocumentKey(string packageId, SearchFilters searchFilters) + { + var lowerId = packageId.ToLowerInvariant(); + var encodedId = EncodeKey(lowerId); + return $"{encodedId}-{GetSearchFilterString(searchFilters)}"; + } + + public static string GetHijackDocumentKey(string packageId, string normalizedVersion) + { + var lowerId = packageId.ToLowerInvariant(); + var lowerVersion = normalizedVersion.ToLowerInvariant(); + return EncodeKey($"{lowerId}/{lowerVersion}"); + } + + public static double GetDownloadScore(double totalDownloadCount) + { + // This score ranges from 0 to less than 100, assuming that the most downloaded + // package has less than 500 million downloads. This scoring function increases + // quickly at first and then becomes approximately linear near the upper bound. + return Math.Sqrt(totalDownloadCount) / 220; + } + + private static string EncodeKey(string rawKey) + { + // First, encode the raw value for uniqueness. + var bytes = Encoding.UTF8.GetBytes(rawKey); + var unique = HttpServerUtility.UrlTokenEncode(bytes); + + // Then, prepend a string as close to the raw key as possible, for readability. + var readable = ReplaceUnsafeKeyCharacters(rawKey).TrimStart('_'); + + return readable.Length > 0 ? $"{readable}-{unique}" : unique; + } + + private static string ReplaceUnsafeKeyCharacters(string input) + { + return Regex.Replace( + input, + "[^A-Za-z0-9-_]", // Remove equal sign as well, since it's ugly. + "_", + RegexOptions.None, + TimeSpan.FromSeconds(30)); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/DownloadTransferrer.cs b/src/NuGet.Services.AzureSearch/DownloadTransferrer.cs new file mode 100644 index 000000000..76a54fbdb --- /dev/null +++ b/src/NuGet.Services.AzureSearch/DownloadTransferrer.cs @@ -0,0 +1,226 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Services.AzureSearch.Auxiliary2AzureSearch; +using NuGet.Services.AzureSearch.AuxiliaryFiles; + +namespace NuGet.Services.AzureSearch +{ + public class DownloadTransferrer : IDownloadTransferrer + { + private readonly IDataSetComparer _dataComparer; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public DownloadTransferrer( + IDataSetComparer dataComparer, + IOptionsSnapshot options, + ILogger logger) + { + _dataComparer = dataComparer ?? throw new ArgumentNullException(nameof(dataComparer)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public SortedDictionary InitializeDownloadTransfers( + DownloadData downloads, + PopularityTransferData outgoingTransfers) + { + // Downloads are transferred from a "from" package to one or more "to" packages. + // The "outgoingTransfers" maps "from" packages to their corresponding "to" packages. + // The "incomingTransfers" maps "to" packages to their corresponding "from" packages. + var incomingTransfers = GetIncomingTransfers(outgoingTransfers); + + // Get the transfer changes for all packages that have popularity transfers. + var packageIds = new HashSet(StringComparer.OrdinalIgnoreCase); + packageIds.UnionWith(outgoingTransfers.Keys); + packageIds.UnionWith(incomingTransfers.Keys); + + return ApplyDownloadTransfers( + downloads, + outgoingTransfers, + incomingTransfers, + packageIds); + } + + public SortedDictionary UpdateDownloadTransfers( + DownloadData downloads, + SortedDictionary downloadChanges, + PopularityTransferData oldTransfers, + PopularityTransferData newTransfers) + { + Guard.Assert( + downloadChanges.Comparer == StringComparer.OrdinalIgnoreCase, + $"Download changes should have comparer {nameof(StringComparer.OrdinalIgnoreCase)}"); + + Guard.Assert( + downloadChanges.All(x => downloads.GetDownloadCount(x.Key) == x.Value), + "The download changes should match the latest downloads"); + + // Downloads are transferred from a "from" package to one or more "to" packages. + // The "oldTransfers" and "newTransfers" maps "from" packages to their corresponding "to" packages. + // The "incomingTransfers" maps "to" packages to their corresponding "from" packages. + var incomingTransfers = GetIncomingTransfers(newTransfers); + + _logger.LogInformation("Detecting changes in popularity transfers."); + var transferChanges = _dataComparer.ComparePopularityTransfers(oldTransfers, newTransfers); + _logger.LogInformation("{Count} popularity transfers have changed.", transferChanges.Count); + + // Get the transfer changes for packages affected by the download and transfer changes. + var affectedPackages = GetPackagesAffectedByChanges( + oldTransfers, + newTransfers, + incomingTransfers, + transferChanges, + downloadChanges); + + return ApplyDownloadTransfers( + downloads, + newTransfers, + incomingTransfers, + affectedPackages); + } + + private SortedDictionary ApplyDownloadTransfers( + DownloadData downloads, + PopularityTransferData outgoingTransfers, + SortedDictionary> incomingTransfers, + HashSet packageIds) + { + _logger.LogInformation( + "{Count} package IDs have download changes due to popularity transfers.", + packageIds.Count); + + var result = new SortedDictionary(StringComparer.OrdinalIgnoreCase); + foreach (var packageId in packageIds) + { + result[packageId] = TransferPackageDownloads( + packageId, + outgoingTransfers, + incomingTransfers, + downloads); + } + + return result; + } + + private SortedDictionary> GetIncomingTransfers( + PopularityTransferData outgoingTransfers) + { + var result = new SortedDictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (var outgoingTransfer in outgoingTransfers) + { + var fromPackage = outgoingTransfer.Key; + + foreach (var toPackage in outgoingTransfer.Value) + { + if (!result.TryGetValue(toPackage, out var incomingTransfer)) + { + incomingTransfer = new SortedSet(StringComparer.OrdinalIgnoreCase); + result.Add(toPackage, incomingTransfer); + } + + incomingTransfer.Add(fromPackage); + } + } + + return result; + } + + private HashSet GetPackagesAffectedByChanges( + PopularityTransferData oldOutgoingTransfers, + PopularityTransferData outgoingTransfers, + SortedDictionary> incomingTransfers, + SortedDictionary transferChanges, + SortedDictionary downloadChanges) + { + var affectedPackages = new HashSet(StringComparer.OrdinalIgnoreCase); + + // If a package adds, changes, or removes outgoing transfers: + // Update "from" package + // Update all new "to" packages + // Update all old "to" packages (in case "to" packages were removed) + foreach (var transferChange in transferChanges) + { + var fromPackage = transferChange.Key; + var toPackages = transferChange.Value; + + affectedPackages.Add(fromPackage); + affectedPackages.UnionWith(toPackages); + + if (oldOutgoingTransfers.TryGetValue(fromPackage, out var oldToPackages)) + { + affectedPackages.UnionWith(oldToPackages); + } + } + + // If a package has download changes and outgoing transfers + // Update "from" package + // Update all "to" packages + // + // If a package has download changes and incoming transfers + // Update "to" package + foreach (var packageId in downloadChanges.Keys) + { + if (outgoingTransfers.TryGetValue(packageId, out var toPackages)) + { + affectedPackages.Add(packageId); + affectedPackages.UnionWith(toPackages); + } + + if (incomingTransfers.ContainsKey(packageId)) + { + affectedPackages.Add(packageId); + } + } + + return affectedPackages; + } + + private long TransferPackageDownloads( + string packageId, + PopularityTransferData outgoingTransfers, + SortedDictionary> incomingTransfers, + DownloadData downloads) + { + var originalDownloads = downloads.GetDownloadCount(packageId); + var transferPercentage = _options.Value.Scoring.PopularityTransfer; + + // Calculate packages with outgoing transfers first. These packages transfer a percentage + // or their downloads equally to a set of "incoming" packages. Packages with both outgoing + // and incoming transfers "reject" the incoming transfers. + if (outgoingTransfers.ContainsKey(packageId)) + { + var keepPercentage = 1 - transferPercentage; + + return (long)(originalDownloads * keepPercentage); + } + + // Next, calculate packages with incoming transfers. These packages receive downloads + // from one or more "outgoing" packages. + if (incomingTransfers.TryGetValue(packageId, out var incomingTransferIds)) + { + var result = originalDownloads; + + foreach (var incomingTransferId in incomingTransferIds) + { + var incomingDownloads = downloads.GetDownloadCount(incomingTransferId); + var incomingSplit = outgoingTransfers[incomingTransferId].Count; + + result += (long)(incomingDownloads * transferPercentage / incomingSplit); + } + + return result; + } + + // The package has no outgoing or incoming transfers. Return its downloads unchanged. + return originalDownloads; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/DurationMeasurement.cs b/src/NuGet.Services.AzureSearch/DurationMeasurement.cs new file mode 100644 index 000000000..c4eac25fb --- /dev/null +++ b/src/NuGet.Services.AzureSearch/DurationMeasurement.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch +{ + public class DurationMeasurement + { + public DurationMeasurement(T result, TimeSpan duration) + { + Value = result; + Duration = duration; + } + + public T Value { get; } + public TimeSpan Duration { get; } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/EntitiesContextFactory.cs b/src/NuGet.Services.AzureSearch/EntitiesContextFactory.cs new file mode 100644 index 000000000..9bf43de5a --- /dev/null +++ b/src/NuGet.Services.AzureSearch/EntitiesContextFactory.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using NuGet.Jobs; +using NuGet.Jobs.Configuration; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch +{ + public class EntitiesContextFactory : IEntitiesContextFactory + { + private readonly ISqlConnectionFactory _connectionFactory; + + public EntitiesContextFactory(ISqlConnectionFactory connectionFactory) + { + _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory)); + } + + public async Task CreateAsync(bool readOnly) + { + var sqlConnection = await _connectionFactory.CreateAsync(); + return new EntitiesContext(sqlConnection, readOnly); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/FeatureFlagService.cs b/src/NuGet.Services.AzureSearch/FeatureFlagService.cs new file mode 100644 index 000000000..8669c4f48 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/FeatureFlagService.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Services.FeatureFlags; + +namespace NuGet.Services.AzureSearch +{ + public class FeatureFlagService : IFeatureFlagService + { + private const string SearchPrefix = "Search."; + + private readonly IFeatureFlagClient _featureFlagClient; + + public FeatureFlagService(IFeatureFlagClient featureFlagClient) + { + _featureFlagClient = featureFlagClient ?? throw new ArgumentNullException(nameof(featureFlagClient)); + } + + public bool IsPopularityTransferEnabled() + { + return _featureFlagClient.IsEnabled(SearchPrefix + "TransferPopularity", defaultValue: true); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/HijackDocumentBuilder.cs b/src/NuGet.Services.AzureSearch/HijackDocumentBuilder.cs new file mode 100644 index 000000000..c7f5d41d8 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/HijackDocumentBuilder.cs @@ -0,0 +1,119 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Protocol.Catalog; +using NuGet.Services.Entities; + +namespace NuGet.Services.AzureSearch +{ + public class HijackDocumentBuilder : IHijackDocumentBuilder + { + private readonly IBaseDocumentBuilder _baseDocumentBuilder; + + public HijackDocumentBuilder(IBaseDocumentBuilder baseDocumentBuilder) + { + _baseDocumentBuilder = baseDocumentBuilder ?? throw new ArgumentNullException(nameof(baseDocumentBuilder)); + } + + public KeyedDocument Keyed( + string packageId, + string normalizedVersion) + { + var document = new KeyedDocument(); + + PopulateKey(document, packageId, normalizedVersion); + + return document; + } + + public HijackDocument.Latest LatestFromCatalog( + string packageId, + string normalizedVersion, + DateTimeOffset lastCommitTimestamp, + string lastCommitId, + HijackDocumentChanges changes) + { + var document = new HijackDocument.Latest(); + + PopulateLatest( + document, + packageId, + normalizedVersion, + lastUpdatedFromCatalog: true, + lastCommitTimestamp: lastCommitTimestamp, + lastCommitId: lastCommitId, + changes: changes); + + return document; + } + + public HijackDocument.Full FullFromCatalog( + string normalizedVersion, + HijackDocumentChanges changes, + PackageDetailsCatalogLeaf leaf) + { + var document = new HijackDocument.Full(); + + PopulateLatest( + document, + leaf.PackageId, + normalizedVersion, + lastUpdatedFromCatalog: true, + lastCommitTimestamp: leaf.CommitTimestamp, + lastCommitId: leaf.CommitId, + changes: changes); + _baseDocumentBuilder.PopulateMetadata(document, normalizedVersion, leaf); + document.Listed = leaf.IsListed(); + + return document; + } + + public HijackDocument.Full FullFromDb( + string packageId, + HijackDocumentChanges changes, + Package package) + { + var document = new HijackDocument.Full(); + + PopulateLatest( + document, + packageId, + lastUpdatedFromCatalog: false, + lastCommitTimestamp: null, + lastCommitId: null, + normalizedVersion: package.NormalizedVersion, + changes: changes); + _baseDocumentBuilder.PopulateMetadata(document, packageId, package); + document.Listed = package.Listed; + + return document; + } + + private void PopulateLatest( + T document, + string packageId, + string normalizedVersion, + bool lastUpdatedFromCatalog, + DateTimeOffset? lastCommitTimestamp, + string lastCommitId, + HijackDocumentChanges changes) where T : KeyedDocument, HijackDocument.ILatest + { + PopulateKey(document, packageId, normalizedVersion); + _baseDocumentBuilder.PopulateCommitted( + document, + lastUpdatedFromCatalog, + lastCommitTimestamp, + lastCommitId); + document.IsLatestStableSemVer1 = changes.LatestStableSemVer1; + document.IsLatestSemVer1 = changes.LatestSemVer1; + document.IsLatestStableSemVer2 = changes.LatestStableSemVer2; + document.IsLatestSemVer2 = changes.LatestSemVer2; + } + + private static void PopulateKey(KeyedDocument document, string packageId, string normalizedVersion) + { + document.Key = DocumentUtilities.GetHijackDocumentKey(packageId, normalizedVersion); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/IAzureSearchCommand.cs b/src/NuGet.Services.AzureSearch/IAzureSearchCommand.cs new file mode 100644 index 000000000..c0c76a371 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/IAzureSearchCommand.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch +{ + public interface IAzureSearchCommand + { + Task ExecuteAsync(); + } +} diff --git a/src/NuGet.Services.AzureSearch/IAzureSearchTelemetryService.cs b/src/NuGet.Services.AzureSearch/IAzureSearchTelemetryService.cs new file mode 100644 index 000000000..e1203b89f --- /dev/null +++ b/src/NuGet.Services.AzureSearch/IAzureSearchTelemetryService.cs @@ -0,0 +1,62 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Services.AzureSearch.SearchService; + +namespace NuGet.Services.AzureSearch +{ + public interface IAzureSearchTelemetryService + { + void TrackAuxiliaryFileDownloaded(string blobName, TimeSpan elapsed); + void TrackAuxiliaryFilesReload(IReadOnlyList reloadedNames, IReadOnlyList notModifiedNames, TimeSpan elapsed); + IDisposable TrackGetLatestLeaves(string packageId, int requestedVersions); + void TrackGetOwnersForPackageId(int ownerCount, TimeSpan elapsed); + void TrackIndexPushFailure(string indexName, int documentCount, TimeSpan elapsed); + void TrackIndexPushSplit(string indexName, int documentCount); + void TrackIndexPushSuccess(string indexName, int documentCount, TimeSpan elapsed); + void TrackUpdateOwnersCompleted(JobOutcome outcome, TimeSpan elapsed); + void TrackOwnerSetComparison(int oldCount, int newCount, int changeCount, TimeSpan elapsed); + void TrackReadLatestIndexedOwners(int packageIdCount, TimeSpan elapsed); + void TrackReadLatestOwnersFromDatabase(int packageIdCount, TimeSpan elapsed); + void TrackPopularityTransfersSetComparison(int oldCount, int newCount, int changeCount, TimeSpan elapsed); + void TrackReadLatestIndexedPopularityTransfers(int? outgoingTransfers, bool modified, TimeSpan elapsed); + void TrackReadLatestPopularityTransfersFromDatabase(int outgoingTransfers, TimeSpan elapsed); + void TrackReadLatestVerifiedPackagesFromDatabase(int packageIdCount, TimeSpan elapsed); + IDisposable TrackReplaceLatestIndexedOwners(int packageIdCount); + IDisposable TrackUploadOwnerChangeHistory(int packageIdCount); + IDisposable TrackReplaceLatestIndexedPopularityTransfers(int outgoingTransfers); + IDisposable TrackVersionListsUpdated(int versionListCount, int workerCount); + IDisposable TrackCatalog2AzureSearchProcessBatch(int catalogLeafCount, int latestCatalogLeafCount, int packageIdCount); + void TrackV2SearchQueryWithSearchIndex(TimeSpan elapsed); + void TrackV2SearchQueryWithHijackIndex(TimeSpan elapsed); + void TrackAutocompleteQuery(TimeSpan elapsed); + void TrackDownloadSetComparison(int oldCount, int newCount, int changeCount, TimeSpan elapsed); + void TrackV3SearchQuery(TimeSpan elapsed); + void TrackGetSearchServiceStatus(SearchStatusOptions options, bool success, TimeSpan elapsed); + void TrackDocumentCountQuery(string indexName, long count, TimeSpan elapsed); + void TrackDownloadCountDecrease( + string packageId, + string version, + bool oldHasId, + bool oldHasVersion, + long oldDownloads, + bool newHasId, + bool newHasVersion, + long newDownloads); + void TrackWarmQuery(string indexName, TimeSpan elapsed); + void TrackLastCommitTimestampQuery(string indexName, DateTimeOffset? lastCommitTimestamp, TimeSpan elapsed); + void TrackReadLatestIndexedDownloads(int? packageIdCount, bool notModified, TimeSpan elapsed); + IDisposable TrackReplaceLatestIndexedDownloads(int packageIdCount); + void TrackAuxiliary2AzureSearchCompleted(JobOutcome outcome, TimeSpan elapsed); + void TrackV3GetDocument(TimeSpan elapsed); + void TrackV2GetDocumentWithSearchIndex(TimeSpan elapsed); + void TrackV2GetDocumentWithHijackIndex(TimeSpan elapsed); + void TrackUpdateVerifiedPackagesCompleted(JobOutcome outcome, TimeSpan elapsed); + void TrackReadLatestVerifiedPackages(int? packageIdCount, bool modified, TimeSpan elapsed); + IDisposable TrackReplaceLatestVerifiedPackages(int packageIdCount); + void TrackAuxiliaryFilesStringCache(int stringCount, long charCount, int requestCount, int hitCount); + void TrackUpdateDownloadsCompleted(JobOutcome outcome, TimeSpan elapsed); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/IBaseDocumentBuilder.cs b/src/NuGet.Services.AzureSearch/IBaseDocumentBuilder.cs new file mode 100644 index 000000000..a794bfac6 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/IBaseDocumentBuilder.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Protocol.Catalog; +using NuGet.Services.Entities; + +namespace NuGet.Services.AzureSearch +{ + public interface IBaseDocumentBuilder + { + void PopulateUpdated( + IUpdatedDocument document, + bool lastUpdatedFromCatalog); + void PopulateCommitted( + ICommittedDocument document, + bool lastUpdatedFromCatalog, + DateTimeOffset? lastCommitTimestamp, + string lastCommitId); + void PopulateMetadata( + IBaseMetadataDocument document, + string packageId, + Package package); + void PopulateMetadata( + IBaseMetadataDocument document, + string normalizedVersion, + PackageDetailsCatalogLeaf leaf); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/IBatchPusher.cs b/src/NuGet.Services.AzureSearch/IBatchPusher.cs new file mode 100644 index 000000000..d6a0a90c3 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/IBatchPusher.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch +{ + /// + /// This is a stateful interface that handles pushing index actions to Azure Search and updating the version list + /// resource based on the enqueued in a batch-wise fashion. This interface is not + /// designed to be thread-safe. + /// + public interface IBatchPusher + { + /// + /// This method does not work with Azure Search or the version lists. It simply enqueues the actions in memory + /// and associates the work with the provided package ID. + /// + /// The package ID related to the index actions. + /// The index actions and version list. + void EnqueueIndexActions(string packageId, IndexActions indexActions); + + /// + /// Pushes full batches to Azure Search based on the based provided to + /// . If there is not enough data to create a full batch, + /// it is not pushed by this method (until enough additional data is enqueued to make a full batch). When all of + /// the index actions for a specific package ID completed (pushed to Azure Search), the corresponding version + /// list is also updated. Hijack index changes are applied before search index changes. Any failures are + /// returned in the result. + /// + Task TryPushFullBatchesAsync(); + + /// + /// Same as but if there is a partial batch remaining, it is also pushed. + /// Any failures are returned in the result. + /// + Task TryFinishAsync(); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/IBlobContainerBuilder.cs b/src/NuGet.Services.AzureSearch/IBlobContainerBuilder.cs new file mode 100644 index 000000000..e85fc7428 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/IBlobContainerBuilder.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch +{ + public interface IBlobContainerBuilder + { + Task DeleteIfExistsAsync(); + Task CreateAsync(bool retryOnConflict); + Task CreateIfNotExistsAsync(); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/IDatabaseAuxiliaryDataFetcher.cs b/src/NuGet.Services.AzureSearch/IDatabaseAuxiliaryDataFetcher.cs new file mode 100644 index 000000000..6b5b443f4 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/IDatabaseAuxiliaryDataFetcher.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGet.Services.AzureSearch.AuxiliaryFiles; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Fetches auxiliary data from the Gallery database. + /// + public interface IDatabaseAuxiliaryDataFetcher + { + /// + /// Fetch the owners for a specific package ID. If the package registration does not exist or if there are no + /// owners, an empty string array is returned. If there are owners, they are sorted. + /// The package ID to fetch owners for. + /// The sorted array of owners. Can be empty but won't ever be null. + Task GetOwnersOrEmptyAsync(string id); + + /// + /// Fetch a mapping from package ID to set of owners for each package registration (i.e. package ID) in the + /// gallery database. + /// + Task>> GetPackageIdToOwnersAsync(); + + /// + /// Fetch a mapping of package IDs to set of replacement package IDs for each renamed packages that transfer + /// popularity in the gallery database. + /// + Task GetPopularityTransfersAsync(); + + /// + /// Fetch the set of all verified package IDs. + /// + Task> GetVerifiedPackagesAsync(); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/IDownloadTransferrer.cs b/src/NuGet.Services.AzureSearch/IDownloadTransferrer.cs new file mode 100644 index 000000000..560e3422f --- /dev/null +++ b/src/NuGet.Services.AzureSearch/IDownloadTransferrer.cs @@ -0,0 +1,51 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using NuGet.Services.AzureSearch.AuxiliaryFiles; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Determines the downloads that should be changed due to popularity transfers. + /// + public interface IDownloadTransferrer + { + /// + /// Determine changes that should be applied to the initial downloads data due to popularity transfers. + /// + /// The initial downloads data. + /// + /// The initial popularity transfers. Maps packages transferring their popularity + /// away to the list of packages receiving the popularity. + /// + /// + /// The downloads that are changed due to transfers. Maps package ids to the new download value. + /// + SortedDictionary InitializeDownloadTransfers( + DownloadData downloads, + PopularityTransferData popularityTransfers); + + /// + /// Determine changes that should be applied to the latest downloads data due to popularity transfers. + /// + /// The latest downloads data. + /// The downloads that have changed since the last index. + /// + /// The previously indexed popularity transfers. Maps packages transferring their popularity + /// away to the list of packages receiving the popularity. + /// + /// + /// The latest popularity transfers. Maps packages transferring their popularity + /// away to the list of packages receiving the popularity. + /// + /// + /// The downloads that are changed due to transfers. Maps package ids to the new download value. + /// + SortedDictionary UpdateDownloadTransfers( + DownloadData downloads, + SortedDictionary downloadChanges, + PopularityTransferData oldTransfers, + PopularityTransferData newTransfers); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/IEntitiesContextFactory.cs b/src/NuGet.Services.AzureSearch/IEntitiesContextFactory.cs new file mode 100644 index 000000000..424a52a17 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/IEntitiesContextFactory.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch +{ + public interface IEntitiesContextFactory + { + Task CreateAsync(bool readOnly); + } +} diff --git a/src/NuGet.Services.AzureSearch/IFeatureFlagService.cs b/src/NuGet.Services.AzureSearch/IFeatureFlagService.cs new file mode 100644 index 000000000..5c038196d --- /dev/null +++ b/src/NuGet.Services.AzureSearch/IFeatureFlagService.cs @@ -0,0 +1,10 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch +{ + public interface IFeatureFlagService + { + bool IsPopularityTransferEnabled(); + } +} diff --git a/src/NuGet.Services.AzureSearch/IHijackDocumentBuilder.cs b/src/NuGet.Services.AzureSearch/IHijackDocumentBuilder.cs new file mode 100644 index 000000000..6515d89b4 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/IHijackDocumentBuilder.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Protocol.Catalog; +using NuGet.Services.Entities; + +namespace NuGet.Services.AzureSearch +{ + public interface IHijackDocumentBuilder + { + KeyedDocument Keyed( + string packageId, + string normalizedVersion); + + HijackDocument.Latest LatestFromCatalog( + string packageId, + string normalizedVersion, + DateTimeOffset lastCommitTimestamp, + string lastCommitId, + HijackDocumentChanges changes); + + HijackDocument.Full FullFromDb( + string packageId, + HijackDocumentChanges changes, + Package package); + + HijackDocument.Full FullFromCatalog( + string normalizedVersion, + HijackDocumentChanges changes, + PackageDetailsCatalogLeaf leaf); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/IIndexBuilder.cs b/src/NuGet.Services.AzureSearch/IIndexBuilder.cs new file mode 100644 index 000000000..a455ad623 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/IIndexBuilder.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch +{ + public interface IIndexBuilder + { + Task CreateHijackIndexAsync(); + Task CreateHijackIndexIfNotExistsAsync(); + Task CreateSearchIndexAsync(); + Task CreateSearchIndexIfNotExistsAsync(); + Task DeleteHijackIndexIfExistsAsync(); + Task DeleteSearchIndexIfExistsAsync(); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/ISearchDocumentBuilder.cs b/src/NuGet.Services.AzureSearch/ISearchDocumentBuilder.cs new file mode 100644 index 000000000..43cad5c92 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/ISearchDocumentBuilder.cs @@ -0,0 +1,71 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Protocol.Catalog; +using NuGet.Services.Entities; + +namespace NuGet.Services.AzureSearch +{ + public interface ISearchDocumentBuilder + { + SearchDocument.LatestFlags LatestFlagsOrNull( + VersionLists versionLists, + SearchFilters searchFilters); + + KeyedDocument Keyed( + string packageId, + SearchFilters searchFilters); + + SearchDocument.UpdateOwners UpdateOwners( + string packageId, + SearchFilters searchFilters, + string[] owners); + + SearchDocument.UpdateDownloadCount UpdateDownloadCount( + string packageId, + SearchFilters searchFilters, + long totalDownloadCount); + + SearchDocument.UpdateVersionList UpdateVersionListFromCatalog( + string packageId, + SearchFilters searchFilters, + DateTimeOffset lastCommitTimestamp, + string lastCommitId, + string[] versions, + bool isLatestStable, + bool isLatest); + + SearchDocument.UpdateVersionListAndOwners UpdateVersionListAndOwnersFromCatalog( + string packageId, + SearchFilters searchFilters, + DateTimeOffset lastCommitTimestamp, + string lastCommitId, + string[] versions, + bool isLatestStable, + bool isLatest, + string[] owners); + + SearchDocument.Full FullFromDb( + string packageId, + SearchFilters searchFilters, + string[] versions, + bool isLatestStable, + bool isLatest, + string fullVersion, + Package package, + string[] owners, + long totalDownloadCount, + bool isExcludedByDefault); + + SearchDocument.UpdateLatest UpdateLatestFromCatalog( + SearchFilters searchFilters, + string[] versions, + bool isLatestStable, + bool isLatest, + string normalizedVersion, + string fullVersion, + PackageDetailsCatalogLeaf leaf, + string[] owners); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/ISearchIndexActionBuilder.cs b/src/NuGet.Services.AzureSearch/ISearchIndexActionBuilder.cs new file mode 100644 index 000000000..6fecfec60 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/ISearchIndexActionBuilder.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Builds a set of index actions for a specific package ID against the search index. No hijack index actions + /// should be returned by this interface. + /// + public interface ISearchIndexActionBuilder + { + /// + /// Generates a set of index actions for all search documents that exist for this package ID. The document + /// for each search filter that is sent to the search index is built by . It is + /// assumed that the caller's implementation of the delegate knows the by context. + /// + /// The package ID to produce documents for. + /// A delegate used to initialize the document. + /// The index actions. + Task UpdateAsync(string packageId, Func buildDocument); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/IndexActions.cs b/src/NuGet.Services.AzureSearch/IndexActions.cs new file mode 100644 index 000000000..19f1ee3c9 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/IndexActions.cs @@ -0,0 +1,32 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch +{ + /// + /// instances separated by whether they apply to the search index or the hijack index + /// as well as the version list data to write to storage after the index actions have been applied. + /// + public class IndexActions + { + public IndexActions( + IReadOnlyList> search, + IReadOnlyList> hijack, + ResultAndAccessCondition versionListDataResult) + { + Search = search ?? throw new ArgumentNullException(nameof(search)); + Hijack = hijack ?? throw new ArgumentNullException(nameof(hijack)); + VersionListDataResult = versionListDataResult ?? throw new ArgumentNullException(nameof(versionListDataResult)); + } + + public IReadOnlyList> Search { get; } + public IReadOnlyList> Hijack { get; } + public ResultAndAccessCondition VersionListDataResult { get; } + + public bool IsEmpty => Search.Count == 0 && Hijack.Count == 0; + } +} diff --git a/src/NuGet.Services.AzureSearch/IndexBuilder.cs b/src/NuGet.Services.AzureSearch/IndexBuilder.cs new file mode 100644 index 000000000..f64e9ec12 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/IndexBuilder.cs @@ -0,0 +1,142 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Azure.Search; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Services.AzureSearch.ScoringProfiles; +using NuGet.Services.AzureSearch.Wrappers; + +namespace NuGet.Services.AzureSearch +{ + public class IndexBuilder : IIndexBuilder + { + private readonly ISearchServiceClientWrapper _serviceClient; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public IndexBuilder( + ISearchServiceClientWrapper serviceClient, + IOptionsSnapshot options, + ILogger logger) + { + _serviceClient = serviceClient ?? throw new ArgumentNullException(nameof(serviceClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task CreateSearchIndexAsync() + { + await CreateIndexAsync(InitializeSearchIndex()); + } + + public async Task CreateHijackIndexAsync() + { + await CreateIndexAsync(InitializeHijackIndex()); + } + + public async Task CreateSearchIndexIfNotExistsAsync() + { + await CreateIndexIfNotExistsAsync(InitializeSearchIndex()); + } + + public async Task CreateHijackIndexIfNotExistsAsync() + { + await CreateIndexIfNotExistsAsync(InitializeHijackIndex()); + } + + public async Task DeleteSearchIndexIfExistsAsync() + { + await DeleteIndexIfExistsAsync(_options.Value.SearchIndexName); + } + + public async Task DeleteHijackIndexIfExistsAsync() + { + await DeleteIndexIfExistsAsync(_options.Value.HijackIndexName); + } + + private async Task DeleteIndexIfExistsAsync(string indexName) + { + if (await _serviceClient.Indexes.ExistsAsync(indexName)) + { + _logger.LogWarning("Deleting index {IndexName}.", indexName); + await _serviceClient.Indexes.DeleteAsync(indexName); + _logger.LogWarning("Done deleting index {IndexName}.", indexName); + } + else + { + _logger.LogInformation("Skipping the deletion of index {IndexName} since it does not exist.", indexName); + } + } + + private async Task CreateIndexAsync(Index index) + { + _logger.LogInformation("Creating index {IndexName}.", index.Name); + await _serviceClient.Indexes.CreateAsync(index); + _logger.LogInformation("Done creating index {IndexName}.", index.Name); + } + + private async Task CreateIndexIfNotExistsAsync(Index index) + { + if (!(await _serviceClient.Indexes.ExistsAsync(index.Name))) + { + await CreateIndexAsync(index); + } + else + { + _logger.LogInformation("Skipping the creation of index {IndexName} since it already exists.", index.Name); + } + } + + private Index InitializeSearchIndex() + { + return InitializeIndex( + _options.Value.SearchIndexName, addScoringProfile: true); + } + + private Index InitializeHijackIndex() + { + return InitializeIndex( + _options.Value.HijackIndexName, addScoringProfile: false); + } + + private Index InitializeIndex(string name, bool addScoringProfile) + { + var index = new Index + { + Name = name, + Fields = FieldBuilder.BuildForType(), + Analyzers = new List + { + DescriptionAnalyzer.Instance, + ExactMatchCustomAnalyzer.Instance, + PackageIdCustomAnalyzer.Instance, + TagsCustomAnalyzer.Instance + }, + Tokenizers = new List + { + PackageIdCustomTokenizer.Instance, + }, + TokenFilters = new List + { + IdentifierCustomTokenFilter.Instance, + TruncateCustomTokenFilter.Instance, + }, + }; + + if (addScoringProfile) + { + var scoringProfile = DefaultScoringProfile.Create(_options.Value.Scoring); + + index.ScoringProfiles = new List { scoringProfile }; + index.DefaultScoringProfile = DefaultScoringProfile.Name; + } + + return index; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/JobOutcome.cs b/src/NuGet.Services.AzureSearch/JobOutcome.cs new file mode 100644 index 000000000..c8d1e4263 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/JobOutcome.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch +{ + public enum JobOutcome + { + Success, + Failure, + NoOp, + } +} diff --git a/src/NuGet.Services.AzureSearch/Measure.cs b/src/NuGet.Services.AzureSearch/Measure.cs new file mode 100644 index 000000000..89777e86e --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Measure.cs @@ -0,0 +1,30 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch +{ + public static class Measure + { + public static async Task> DurationWithValueAsync(Func> actAsync) + { + var stopwatch = Stopwatch.StartNew(); + var value = await actAsync(); + stopwatch.Stop(); + return new DurationMeasurement(value, stopwatch.Elapsed); + } + + public static async Task DurationAsync(Func actAsync) + { + var result = await DurationWithValueAsync(async () => + { + await actAsync(); + return 0; + }); + return result.Duration; + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Models/BaseMetadataDocument.cs b/src/NuGet.Services.AzureSearch/Models/BaseMetadataDocument.cs new file mode 100644 index 000000000..cd425f34c --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Models/BaseMetadataDocument.cs @@ -0,0 +1,89 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Azure.Search; +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch +{ + public abstract class BaseMetadataDocument : CommittedDocument, IBaseMetadataDocument + { + [IsFilterable] + [JsonProperty(NullValueHandling = NullValueHandling.Include)] + public int? SemVerLevel { get; set; } + + [IsSearchable] + [Analyzer(DescriptionAnalyzer.Name)] + public string Authors { get; set; } + + public string Copyright { get; set; } + + [IsSortable] + public DateTimeOffset? Created { get; set; } + + [IsSearchable] + [Analyzer(DescriptionAnalyzer.Name)] + public string Description { get; set; } + + public long? FileSize { get; set; } + public string FlattenedDependencies { get; set; } + public string Hash { get; set; } + public string HashAlgorithm { get; set; } + public string IconUrl { get; set; } + public string Language { get; set; } + + [IsSortable] + public DateTimeOffset? LastEdited { get; set; } + + public string LicenseUrl { get; set; } + public string MinClientVersion { get; set; } + + [IsSearchable] + [Analyzer(ExactMatchCustomAnalyzer.Name)] + public string NormalizedVersion { get; set; } + + public string OriginalVersion { get; set; } + + /// + /// The package's identifier. Supports case insensitive exact matching. + /// + [IsSearchable] + [Analyzer(ExactMatchCustomAnalyzer.Name)] + public string PackageId { get; set; } + + [IsFilterable] + public bool? Prerelease { get; set; } + + public string ProjectUrl { get; set; } + + [IsSortable] + [IsFilterable] + public DateTimeOffset? Published { get; set; } + + public string ReleaseNotes { get; set; } + public bool? RequiresLicenseAcceptance { get; set; } + + [IsSortable] + public string SortableTitle { get; set; } + + [IsSearchable] + [Analyzer(DescriptionAnalyzer.Name)] + public string Summary { get; set; } + + [IsSearchable] + [Analyzer(TagsCustomAnalyzer.Name)] + public string[] Tags { get; set; } + + [IsSearchable] + [Analyzer(DescriptionAnalyzer.Name)] + public string Title { get; set; } + + /// + /// The package's identifier. Supports tokenized search. + /// + [IsSearchable] + [Analyzer(PackageIdCustomAnalyzer.Name)] + public string TokenizedPackageId { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Models/CommittedDocument.cs b/src/NuGet.Services.AzureSearch/Models/CommittedDocument.cs new file mode 100644 index 000000000..ae1e7eee2 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Models/CommittedDocument.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Azure.Search; +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch +{ + public abstract class CommittedDocument : UpdatedDocument, ICommittedDocument + { + [IsSortable] + [JsonProperty(NullValueHandling = NullValueHandling.Include)] + public DateTimeOffset? LastCommitTimestamp { get; set; } + + [JsonProperty(NullValueHandling = NullValueHandling.Include)] + public string LastCommitId { get; set; } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Models/CurrentTimestamp.cs b/src/NuGet.Services.AzureSearch/Models/CurrentTimestamp.cs new file mode 100644 index 000000000..0e2a6cb72 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Models/CurrentTimestamp.cs @@ -0,0 +1,60 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; + +namespace NuGet.Services.AzureSearch +{ + /// + /// A "last updated" timestamp set on an Azure Search document is approximate since there is some undefined time + /// between when the document is initialized, then serialized, then pushed to Azure Search, then applied to the + /// index, then made available in a search result. The only period of time that we can mitigate is time between + /// initializing the document and when it is serialized to be sent to Azure Search. To do this, we introduce this + /// stateful type that can capture the current timestamp in when it is next read. This is so + /// the current timestamp is captured as we are serializing the document to JSON to be sent to the Azure Search + /// REST API. + /// + public class CurrentTimestamp + { + public const int FalseInt = 0; + public const int TrueInt = 1; + + private int _setOnNextRead = FalseInt; + private DateTimeOffset? _value; + + /// + /// Defaults to null. After is called, the next time the getter of + /// is the current timestamp is captured then returned. The setter can be used to set the + /// the value but does not undo any calls to . In other words, if is + /// called, then is set to an arbitrary timestamp, the the getter is called, the current + /// timestamp will still be captured. + /// + public DateTimeOffset? Value + { + get + { + var existingValue = Interlocked.CompareExchange( + ref _setOnNextRead, + FalseInt, + TrueInt); + if (existingValue == TrueInt) + { + _value = DateTimeOffset.UtcNow; + } + + return _value; + } + + set + { + _value = value; + } + } + + public void SetOnNextRead() + { + _setOnNextRead = TrueInt; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Models/HijackDocument.cs b/src/NuGet.Services.AzureSearch/Models/HijackDocument.cs new file mode 100644 index 000000000..c26acc65e --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Models/HijackDocument.cs @@ -0,0 +1,54 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Azure.Search; +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch +{ + /// + /// The different models for reading from and writing to the hijack index. + /// + public static class HijackDocument + { + /// + /// All fields available in the hijack index. Used for reading the index and updating a document when + /// is true. + /// + [SerializePropertyNamesAsCamelCase] + public class Full : BaseMetadataDocument, ILatest, IBaseMetadataDocument + { + [IsFilterable] + public bool? Listed { get; set; } + + public bool? IsLatestStableSemVer1 { get; set; } + public bool? IsLatestSemVer1 { get; set; } + public bool? IsLatestStableSemVer2 { get; set; } + public bool? IsLatestSemVer2 { get; set; } + } + + /// + /// Used for updating a document when is false + /// and is false. + /// + [SerializePropertyNamesAsCamelCase] + public class Latest : CommittedDocument, ILatest + { + public bool? IsLatestStableSemVer1 { get; set; } + public bool? IsLatestSemVer1 { get; set; } + public bool? IsLatestStableSemVer2 { get; set; } + public bool? IsLatestSemVer2 { get; set; } + } + + /// + /// Allows index updating code to update the latest booleans. + /// + public interface ILatest : ICommittedDocument + { + bool? IsLatestStableSemVer1 { get; set; } + bool? IsLatestSemVer1 { get; set; } + bool? IsLatestStableSemVer2 { get; set; } + bool? IsLatestSemVer2 { get; set; } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Models/IBaseMetadataDocument.cs b/src/NuGet.Services.AzureSearch/Models/IBaseMetadataDocument.cs new file mode 100644 index 000000000..7ae2587e4 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Models/IBaseMetadataDocument.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch +{ + /// + /// The fields shared between the search index and hijack index. + /// + public interface IBaseMetadataDocument : ICommittedDocument + { + string Authors { get; set; } + string Copyright { get; set; } + DateTimeOffset? Created { get; set; } + string Description { get; set; } + long? FileSize { get; set; } + string FlattenedDependencies { get; set; } + string Hash { get; set; } + string HashAlgorithm { get; set; } + string IconUrl { get; set; } + string Language { get; set; } + DateTimeOffset? LastEdited { get; set; } + string LicenseUrl { get; set; } + string MinClientVersion { get; set; } + string NormalizedVersion { get; set; } + string OriginalVersion { get; set; } + string PackageId { get; set; } + string TokenizedPackageId { get; set; } + bool? Prerelease { get; set; } + string ProjectUrl { get; set; } + DateTimeOffset? Published { get; set; } + string ReleaseNotes { get; set; } + bool? RequiresLicenseAcceptance { get; set; } + int? SemVerLevel { get; set; } + string SortableTitle { get; set; } + string Summary { get; set; } + string[] Tags { get; set; } + string Title { get; set; } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Models/ICommittedDocument.cs b/src/NuGet.Services.AzureSearch/Models/ICommittedDocument.cs new file mode 100644 index 000000000..651f732aa --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Models/ICommittedDocument.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch +{ + /// + /// A document that has data committed from the catalog. + /// + public interface ICommittedDocument : IUpdatedDocument + { + DateTimeOffset? LastCommitTimestamp { get; set; } + string LastCommitId { get; set; } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Models/IKeyedDocument.cs b/src/NuGet.Services.AzureSearch/Models/IKeyedDocument.cs new file mode 100644 index 000000000..3f2934572 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Models/IKeyedDocument.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch +{ + /// + /// A base interface for referring to documents by their key. + /// + public interface IKeyedDocument + { + string Key { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Models/IUpdatedDocument.cs b/src/NuGet.Services.AzureSearch/Models/IUpdatedDocument.cs new file mode 100644 index 000000000..787d9b981 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Models/IUpdatedDocument.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch +{ + /// + /// A document that been updated. + /// + public interface IUpdatedDocument : IKeyedDocument + { + DateTimeOffset? LastUpdatedDocument { get; set; } + string LastDocumentType { get; set; } + bool? LastUpdatedFromCatalog { get; set; } + void SetLastUpdatedDocumentOnNextRead(); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Models/KeyedDocument.cs b/src/NuGet.Services.AzureSearch/Models/KeyedDocument.cs new file mode 100644 index 000000000..a76d404f4 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Models/KeyedDocument.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.ComponentModel.DataAnnotations; +using Microsoft.Azure.Search; +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch +{ + /// + /// This is a base type but can be used directly for operations that only require a document key, such as deleting + /// a document. + /// + [SerializePropertyNamesAsCamelCase] + public class KeyedDocument : IKeyedDocument + { + /// + /// This field is filterable and sortable so that the index can be reliably enumerated for diagnostic purposes. + /// + [Key] + [IsFilterable] + [IsSortable] + public string Key { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Models/SearchDocument.cs b/src/NuGet.Services.AzureSearch/Models/SearchDocument.cs new file mode 100644 index 000000000..09b3af71a --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Models/SearchDocument.cs @@ -0,0 +1,158 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Azure.Search; +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch +{ + /// + /// The different models for reading from and writing to the search index. + /// + public static class SearchDocument + { + /// + /// All fields available in the search index. Used for reading the index and updating the index from database, + /// which has all fields available (as opposed to the catalog, which does not have all fields, like total + /// download count). + /// + [SerializePropertyNamesAsCamelCase] + public class Full : UpdateLatest, IDownloadCount, IIsExcludedByDefault + { + [IsFilterable] + [IsSortable] + public long? TotalDownloadCount { get; set; } + + [IsFilterable] + public double? DownloadScore { get; set; } + + [IsFilterable] + public bool? IsExcludedByDefault { get; set; } + } + + /// + /// Used when processing , + /// or . + /// + [SerializePropertyNamesAsCamelCase] + public class UpdateLatest : BaseMetadataDocument, IVersions, IOwners + { + [IsSearchable] + [Analyzer(ExactMatchCustomAnalyzer.Name)] + public string[] Owners { get; set; } + + [IsFilterable] + public string SearchFilters { get; set; } + + [IsFilterable] + public string[] FilterablePackageTypes { get; set; } + + public string FullVersion { get; set; } + public string[] Versions { get; set; } + public string[] PackageTypes { get; set; } + public bool? IsLatestStable { get; set; } + public bool? IsLatest { get; set; } + } + + /// + /// Used when processing and the owner information has + /// been already been fetched for the purposes of . Note that this model does not + /// need any analyzer or other Azure Search attributes since it is not used for index creation. The + /// and its parent classes handle this. + /// + [SerializePropertyNamesAsCamelCase] + public class UpdateVersionListAndOwners : UpdateVersionList, IOwners + { + public string[] Owners { get; set; } + } + + /// + /// Used when processing . + /// + [SerializePropertyNamesAsCamelCase] + public class UpdateVersionList : CommittedDocument, IVersions + { + public string[] Versions { get; set; } + public bool? IsLatestStable { get; set; } + public bool? IsLatest { get; set; } + } + + /// + /// Used when updating just the owners of a document. Note that this model does not need any analyzer or + /// other Azure Search attributes since it is not used for index creation. The and its + /// parent classes handle this. + /// + [SerializePropertyNamesAsCamelCase] + public class UpdateOwners : UpdatedDocument, IOwners + { + public string[] Owners { get; set; } + } + + /// + /// Used when updating just the fields related to the download count of a document. Note that this model does + /// not need any analyzer or other Azure Search attributes since it is not used for index creation. The + /// and its parent classes handle this. + /// + [SerializePropertyNamesAsCamelCase] + public class UpdateDownloadCount : UpdatedDocument, IDownloadCount + { + public long? TotalDownloadCount { get; set; } + public double? DownloadScore { get; set; } + } + + /// + /// Allows index updating code to apply a new version list to a document. + /// + public interface IVersions : ICommittedDocument + { + string[] Versions { get; set; } + bool? IsLatestStable { get; set; } + bool? IsLatest { get; set; } + } + + /// + /// Allows index updating code to apply a new list of owners to a document. + /// + public interface IOwners : IUpdatedDocument + { + string[] Owners { get; set; } + } + + /// + /// Allows index updating code to apply new download count information to a document. + /// + public interface IDownloadCount : IUpdatedDocument + { + long? TotalDownloadCount { get; set; } + double? DownloadScore { get; set; } + } + + /// + /// Allows index updating code to apply default search exclusion information to a document. + /// + public interface IIsExcludedByDefault: IUpdatedDocument + { + bool? IsExcludedByDefault { get; set; } + } + + /// + /// The data required to populate and other classes. + /// This information, as with all other types under , are specific to a single + /// . That is, the latest version and its metadata given a filtered set of versions + /// per package ID. + /// + public class LatestFlags + { + public LatestFlags(LatestVersionInfo latest, bool isLatestStable, bool isLatest) + { + LatestVersionInfo = latest; + IsLatestStable = isLatestStable; + IsLatest = isLatest; + } + + public LatestVersionInfo LatestVersionInfo { get; } + public bool IsLatestStable { get; } + public bool IsLatest { get; } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Models/UpdatedDocument.cs b/src/NuGet.Services.AzureSearch/Models/UpdatedDocument.cs new file mode 100644 index 000000000..5a55ba174 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Models/UpdatedDocument.cs @@ -0,0 +1,31 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch +{ + public abstract class UpdatedDocument : KeyedDocument + { + private readonly CurrentTimestamp _lastUpdatedDocument = new CurrentTimestamp(); + + [JsonProperty(NullValueHandling = NullValueHandling.Include)] + public DateTimeOffset? LastUpdatedDocument + { + get => _lastUpdatedDocument.Value; + set => _lastUpdatedDocument.Value = value; + } + + [JsonProperty(NullValueHandling = NullValueHandling.Include)] + public string LastDocumentType { get; set; } + + [JsonProperty(NullValueHandling = NullValueHandling.Include)] + public bool? LastUpdatedFromCatalog { get; set; } + + public void SetLastUpdatedDocumentOnNextRead() + { + _lastUpdatedDocument.SetOnNextRead(); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj b/src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj new file mode 100644 index 000000000..6952d5505 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/NuGet.Services.AzureSearch.csproj @@ -0,0 +1,291 @@ + + + + + + Debug + AnyCPU + {1A53FE3D-8041-4773-942F-D73AEF5B82B2} + Library + Properties + NuGet.Services.AzureSearch + NuGet.Services.AzureSearch + v4.7.2 + 512 + true + true + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + .NET Foundation + https://github.com/NuGet/NuGet.Services.Metadata/blob/master/LICENSE + https://github.com/NuGet/NuGet.Services.Metadata + Push NuGetGallery DB packages or catalog leaves to Azure Search. + nuget azure search catalog leaf details incremental collector + Copyright .NET Foundation + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0.3.0 + runtime; build; native; contentfiles; analyzers + all + + + 5.0.3 + + + 4.8.0 + runtime; build; native; contentfiles; analyzers + all + + + + + {e97f23b8-ecb0-4afa-b00c-015c39395fef} + NuGet.Services.Metadata.Catalog + + + {4b4b1efb-8f33-42e6-b79f-54e7f3293d31} + NuGet.Jobs.Common + + + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} + NuGet.Protocol.Catalog + + + {c3f9a738-9759-4b2b-a50d-6507b28a659b} + NuGet.Services.V3 + + + {FA87D075-A934-4443-8D0B-5DB32640B6D7} + Validation.Common.Job + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + + \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/PackageIdToOwnersBuilder.cs b/src/NuGet.Services.AzureSearch/PackageIdToOwnersBuilder.cs new file mode 100644 index 000000000..3b06d3366 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/PackageIdToOwnersBuilder.cs @@ -0,0 +1,70 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.AzureSearch +{ + public class PackageIdToOwnersBuilder + { + private readonly ILogger _logger; + private int _addCount; + private readonly Dictionary _usernameToUsername; + private readonly SortedDictionary> _result; + + public PackageIdToOwnersBuilder(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _addCount = 0; + _usernameToUsername = new Dictionary(StringComparer.OrdinalIgnoreCase); + _result = new SortedDictionary>(StringComparer.OrdinalIgnoreCase); + } + + public void Add(string id, IReadOnlyList owners) + { + foreach (var username in owners) + { + Add(id, username); + } + } + + public void Add(string id, string username) + { + _addCount++; + if (_addCount % 10000 == 0) + { + _logger.LogInformation("{AddCount} ownership records have been added so far.", _addCount); + } + + // Use a single instance of each username string. + if (!_usernameToUsername.TryGetValue(username, out var existingUsername)) + { + _usernameToUsername.Add(username, username); + } + else + { + username = existingUsername; + } + + if (!_result.TryGetValue(id, out var owners)) + { + owners = new SortedSet(StringComparer.OrdinalIgnoreCase); + _result.Add(id, owners); + } + + owners.Add(username); + } + + public SortedDictionary> GetResult() + { + _logger.LogInformation("{RecordCount} ownership records were found.", _addCount); + _logger.LogInformation("{UsernameCount} usernames were found.", _usernameToUsername.Count); + _logger.LogInformation("{IdCount} package IDs were found.", _result.Count); + + return _result; + } + } +} + diff --git a/src/NuGet.Services.AzureSearch/Properties/AssemblyInfo.cs b/src/NuGet.Services.AzureSearch/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..3d8e7112e --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Properties/AssemblyInfo.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Services.AzureSearch")] +[assembly: ComVisible(false)] +[assembly: Guid("1a53fe3d-8041-4773-942f-d73aef5b82b2")] + +#if SIGNED_BUILD +[assembly: InternalsVisibleTo("NuGet.Services.AzureSearch.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100b5fc90e7027f67871e773a8fde8938c81dd402ba65b9201d60593e96c492651e889cc13f1415ebb53fac1131ae0bd333c5ee6021672d9718ea31a8aebd0da0072f25d87dba6fc90ffd598ed4da35e44c398c454307e8e33b8426143daec9f596836f97c8f74750e5975c64e2189f45def46b2a2b1247adc3652bf5c308055da9")] +#else +[assembly: InternalsVisibleTo("NuGet.Services.AzureSearch.Tests")] +#endif diff --git a/src/NuGet.Services.AzureSearch/ScoringProfiles/DefaultScoringProfile.cs b/src/NuGet.Services.AzureSearch/ScoringProfiles/DefaultScoringProfile.cs new file mode 100644 index 000000000..34b50df0c --- /dev/null +++ b/src/NuGet.Services.AzureSearch/ScoringProfiles/DefaultScoringProfile.cs @@ -0,0 +1,93 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Azure.Search.Models; +using NuGet.Services.AzureSearch.SearchService; + +namespace NuGet.Services.AzureSearch.ScoringProfiles +{ + public static class DefaultScoringProfile + { + public const string Name = "nuget_scoring_profile"; + + private static readonly Lazy> KnownSearchIndexFields = new Lazy>(() => + { + Dictionary FieldValues(Type type) + { + return type + .GetFields() + .Where(f => f.IsStatic) + .Where(f => f.IsPublic) + .Where(f => f.FieldType == typeof(string)) + .ToDictionary( + f => f.Name, + f => (string)f.GetValue(null)); + } + + var indexFields = FieldValues(typeof(IndexFields)); + var searchFields = FieldValues(typeof(IndexFields.Search)); + + return indexFields.Concat(searchFields).ToDictionary(x => x.Key, x => x.Value); + }); + + public static ScoringProfile Create(AzureSearchScoringConfiguration config) + { + if (config == null) + { + throw new ArgumentNullException(nameof(config)); + } + + if (config.DownloadScoreBoost <= 1) + { + throw new ArgumentOutOfRangeException(nameof(config.DownloadScoreBoost)); + } + + if (config.FieldWeights.Count != 0) + { + var unknownField = config + .FieldWeights + .Keys + .FirstOrDefault(f => !KnownSearchIndexFields.Value.Keys.Contains(f)); + + if (unknownField != null) + { + throw new ArgumentException( + $"Unknown field '{unknownField}' in {nameof(AzureSearchScoringConfiguration)}.{nameof(config.FieldWeights)}", + nameof(config)); + } + } + + var fieldWeights = config + .FieldWeights + .ToDictionary( + g => KnownSearchIndexFields.Value[g.Key], + g => g.Value); + + return new ScoringProfile( + Name, + textWeights: new TextWeights(fieldWeights), + functions: new List() + { + // Greatly boost results with high download counts. We score off the log of the download count + // with linear interpolation so that the boost slows down at higher download counts. We cannot + // use the raw download count with a log interpolation as that would result in a large boosting + // range, which would need to be offset by an unmanageably high boosting factor. + new MagnitudeScoringFunction( + fieldName: IndexFields.Search.DownloadScore, + boost: config.DownloadScoreBoost, + parameters: new MagnitudeScoringParameters( + boostingRangeStart: 0, + boostingRangeEnd: DocumentUtilities.GetDownloadScore(999_999_999_999), + shouldBoostBeyondRangeByConstant: true), + interpolation: ScoringFunctionInterpolation.Linear), + }, + + // The scores of each Scoring Function should be summed together before multiplying the base relevance scores. + // See: https://stackoverflow.com/questions/41427940/how-do-scoring-profiles-generate-scores-in-azure-search + functionAggregation: ScoringFunctionAggregation.Sum); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchDocumentBuilder.cs b/src/NuGet.Services.AzureSearch/SearchDocumentBuilder.cs new file mode 100644 index 000000000..a67700f8c --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchDocumentBuilder.cs @@ -0,0 +1,361 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using NuGet.Protocol.Catalog; +using NuGet.Services.Entities; + +namespace NuGet.Services.AzureSearch +{ + public class SearchDocumentBuilder : ISearchDocumentBuilder + { + private readonly string[] ImpliedDependency = new string[] { "Dependency" }; + private readonly string[] FilterableImpliedDependency = new string[] { "dependency" }; + + private readonly IBaseDocumentBuilder _baseDocumentBuilder; + + public SearchDocumentBuilder(IBaseDocumentBuilder baseDocumentBuilder) + { + _baseDocumentBuilder = baseDocumentBuilder ?? throw new ArgumentNullException(nameof(baseDocumentBuilder)); + } + + public SearchDocument.LatestFlags LatestFlagsOrNull(VersionLists versionLists, SearchFilters searchFilters) + { + var latest = versionLists.GetLatestVersionInfoOrNull(searchFilters); + if (latest == null) + { + return null; + } + + // The latest version, given the "include prerelease" bit of the search filter, may or may not be the + // absolute latest version when considering both prerelease and stable versions. Consider the following + // cases: + // + // Case #1: + // SearchFilters.Default: + // All versions: 1.0.0, 2.0.0-alpha + // Latest version given filters: 1.0.0 + // V2 search document flags: + // IsLatestStable = true + // IsLatest = false + // + // Case #2: + // SearchFilters.Default: + // All versions: 1.0.0 + // Latest version given filters: 1.0.0 + // V2 search document flags: + // IsLatestStable = true + // IsLatest = true + // + // Case #3: + // SearchFilters.IncludePrerelease: + // All versions: 1.0.0, 2.0.0-alpha + // Latest version given filters: 2.0.0-alpha + // V2 search document flags: + // IsLatestStable = false + // IsLatest = true + // + // Case #4: + // SearchFilters.IncludePrerelease: + // All versions: 1.0.0 + // Latest version given filters: 1.0.0 + // V2 search document flags: + // IsLatestStable = true + // IsLatest = true + // + // In cases #1 and #2, we know the value of IsLatestStable will always be true. We cannot know whether + // IsLatest is true or false without looking at the version list that includes prerelease versions. For + // cases #3 and #4, we know IsLatest will always be true and we can determine IsLatestStable by looking + // at whether the latest version is prerelease or not. + bool isLatestStable; + bool isLatest; + if ((searchFilters & SearchFilters.IncludePrerelease) == 0) + { + // This is the case where prerelease versions are excluded. + var latestIncludePrerelease = versionLists + .GetLatestVersionInfoOrNull(searchFilters | SearchFilters.IncludePrerelease); + Guard.Assert( + latestIncludePrerelease != null, + "If a search filter excludes prerelease and has a latest version, then there is a latest version including prerelease."); + isLatestStable = true; + isLatest = latestIncludePrerelease.ParsedVersion == latest.ParsedVersion; + } + else + { + // This is the case where prerelease versions are included. + isLatestStable = !latest.ParsedVersion.IsPrerelease; + isLatest = true; + } + + return new SearchDocument.LatestFlags(latest, isLatestStable, isLatest); + } + + public KeyedDocument Keyed( + string packageId, + SearchFilters searchFilters) + { + var document = new KeyedDocument(); + + PopulateKey(document, packageId, searchFilters); + + return document; + } + + public SearchDocument.UpdateVersionList UpdateVersionListFromCatalog( + string packageId, + SearchFilters searchFilters, + DateTimeOffset lastCommitTimestamp, + string lastCommitId, + string[] versions, + bool isLatestStable, + bool isLatest) + { + var document = new SearchDocument.UpdateVersionList(); + + PopulateVersions( + document, + packageId, + searchFilters, + lastUpdatedFromCatalog: true, + lastCommitTimestamp: lastCommitTimestamp, + lastCommitId: lastCommitId, + versions: versions, + isLatestStable: isLatestStable, + isLatest: isLatest); + + return document; + } + + public SearchDocument.UpdateVersionListAndOwners UpdateVersionListAndOwnersFromCatalog( + string packageId, + SearchFilters searchFilters, + DateTimeOffset lastCommitTimestamp, + string lastCommitId, + string[] versions, + bool isLatestStable, + bool isLatest, + string[] owners) + { + var document = new SearchDocument.UpdateVersionListAndOwners(); + + PopulateVersions( + document, + packageId, + searchFilters, + lastUpdatedFromCatalog: true, + lastCommitTimestamp: lastCommitTimestamp, + lastCommitId: lastCommitId, + versions: versions, + isLatestStable: isLatestStable, + isLatest: isLatest); + PopulateOwners( + document, + owners); + + return document; + } + + public SearchDocument.UpdateLatest UpdateLatestFromCatalog( + SearchFilters searchFilters, + string[] versions, + bool isLatestStable, + bool isLatest, + string normalizedVersion, + string fullVersion, + PackageDetailsCatalogLeaf leaf, + string[] owners) + { + var document = new SearchDocument.UpdateLatest(); + + // Determine if we have packageTypes to forward. + // Otherwise, we need to let the system know that there were no explicit package types + var packageTypes = leaf.PackageTypes != null && leaf.PackageTypes.Count > 0 ? + leaf.PackageTypes.Select(pt => pt.Name).ToArray() : + null; + + PopulateUpdateLatest( + document, + leaf.PackageId, + searchFilters, + lastUpdatedFromCatalog: true, + lastCommitTimestamp: leaf.CommitTimestamp, + lastCommitId: leaf.CommitId, + versions: versions, + isLatestStable: isLatestStable, + isLatest: isLatest, + fullVersion: fullVersion, + owners: owners, + packageTypes: packageTypes); + _baseDocumentBuilder.PopulateMetadata(document, normalizedVersion, leaf); + + return document; + } + + public SearchDocument.Full FullFromDb( + string packageId, + SearchFilters searchFilters, + string[] versions, + bool isLatestStable, + bool isLatest, + string fullVersion, + Package package, + string[] owners, + long totalDownloadCount, + bool isExcludedByDefault) + { + var document = new SearchDocument.Full(); + + // Determine if we have packageTypes to forward. + // Otherwise, we need to let the system know that there were no explicit package types + var packageTypes = package.PackageTypes != null && package.PackageTypes.Count > 0 ? + package.PackageTypes.Select(pt => pt.Name).ToArray() : + null; + + PopulateUpdateLatest( + document, + packageId, + searchFilters, + lastUpdatedFromCatalog: false, + lastCommitTimestamp: null, + lastCommitId: null, + versions: versions, + isLatestStable: isLatestStable, + isLatest: isLatest, + fullVersion: fullVersion, + owners: owners, + packageTypes: packageTypes); + _baseDocumentBuilder.PopulateMetadata(document, packageId, package); + PopulateDownloadCount(document, totalDownloadCount); + PopulateIsExcludedByDefault(document, isExcludedByDefault); + + return document; + } + + private void PopulateVersions( + T document, + string packageId, + SearchFilters searchFilters, + bool lastUpdatedFromCatalog, + DateTimeOffset? lastCommitTimestamp, + string lastCommitId, + string[] versions, + bool isLatestStable, + bool isLatest) where T : KeyedDocument, SearchDocument.IVersions + { + PopulateKey(document, packageId, searchFilters); + _baseDocumentBuilder.PopulateCommitted( + document, + lastUpdatedFromCatalog, + lastCommitTimestamp, + lastCommitId); + document.Versions = versions; + document.IsLatestStable = isLatestStable; + document.IsLatest = isLatest; + } + + private static void PopulateKey(KeyedDocument document, string packageId, SearchFilters searchFilters) + { + document.Key = DocumentUtilities.GetSearchDocumentKey(packageId, searchFilters); + } + + private void PopulateUpdateLatest( + SearchDocument.UpdateLatest document, + string packageId, + SearchFilters searchFilters, + bool lastUpdatedFromCatalog, + DateTimeOffset? lastCommitTimestamp, + string lastCommitId, + string[] versions, + bool isLatestStable, + bool isLatest, + string fullVersion, + string[] owners, + string[] packageTypes) + { + PopulateVersions( + document, + packageId, + searchFilters, + lastUpdatedFromCatalog, + lastCommitTimestamp, + lastCommitId, + versions, + isLatestStable, + isLatest); + document.SearchFilters = DocumentUtilities.GetSearchFilterString(searchFilters); + document.FullVersion = fullVersion; + + // If the package has explicit types, we will set them here. + // Otherwise, we will treat the package as a "Depedency" type and fill in the explicit type. + if (packageTypes != null && packageTypes.Length > 0) + { + document.PackageTypes = packageTypes; + document.FilterablePackageTypes = packageTypes.Select(pt => pt.ToLowerInvariant()).ToArray(); + } + else + { + document.PackageTypes = ImpliedDependency; + document.FilterablePackageTypes = FilterableImpliedDependency; + } + + PopulateOwners( + document, + owners); + } + + private static void PopulateOwners( + T document, + string[] owners) where T : KeyedDocument, SearchDocument.IOwners + { + document.Owners = owners; + } + + public SearchDocument.UpdateOwners UpdateOwners( + string packageId, + SearchFilters searchFilters, + string[] owners) + { + var document = new SearchDocument.UpdateOwners(); + + PopulateKey(document, packageId, searchFilters); + _baseDocumentBuilder.PopulateUpdated( + document, + lastUpdatedFromCatalog: false); + PopulateOwners(document, owners); + + return document; + } + + public SearchDocument.UpdateDownloadCount UpdateDownloadCount( + string packageId, + SearchFilters searchFilters, + long totalDownloadCount) + { + var document = new SearchDocument.UpdateDownloadCount(); + + PopulateKey(document, packageId, searchFilters); + _baseDocumentBuilder.PopulateUpdated( + document, + lastUpdatedFromCatalog: false); + PopulateDownloadCount(document, totalDownloadCount); + + return document; + } + + private static void PopulateDownloadCount( + T document, + long totalDownloadCount) where T : KeyedDocument, SearchDocument.IDownloadCount + { + document.TotalDownloadCount = totalDownloadCount; + document.DownloadScore = DocumentUtilities.GetDownloadScore(totalDownloadCount); + } + + private static void PopulateIsExcludedByDefault( + T document, + bool isExcludedByDefault) where T : KeyedDocument, SearchDocument.IIsExcludedByDefault + { + document.IsExcludedByDefault = isExcludedByDefault; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchIndexActionBuilder.cs b/src/NuGet.Services.AzureSearch/SearchIndexActionBuilder.cs new file mode 100644 index 000000000..a8b0d01bf --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchIndexActionBuilder.cs @@ -0,0 +1,87 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.AzureSearch +{ + public class SearchIndexActionBuilder : ISearchIndexActionBuilder + { + private readonly IVersionListDataClient _versionListDataClient; + private readonly ILogger _logger; + + public SearchIndexActionBuilder( + IVersionListDataClient versionListDataClient, + ILogger logger) + { + _versionListDataClient = versionListDataClient ?? throw new ArgumentNullException(nameof(versionListDataClient)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task UpdateAsync(string packageId, Func buildDocument) + { + var versionListDataResult = await _versionListDataClient.ReadAsync(packageId); + var versionLists = new VersionLists(versionListDataResult.Result); + + /// Update all of the search documents that exist for this package ID with the provided document builder. + /// Here are some examples of different search filter combinations that could occur. + /// + /// Example #1: 1.0.0 (listed) + /// + /// A stable SemVer 1.0.0 package matches all search filters, so one index action will be produced for + /// each search document. That is four in total. All of these search documents have the same latest + /// version: 1.0.0. + /// + /// Example #2: 1.0.0 (unlisted), 2.0.0 (unlisted) + /// + /// There are no search documents at all in this case since there is no listed version. No index actions + /// are produced in this case. + /// + /// Example #3: 1.0.0-beta (listed), 2.0.0-beta.1 (listed) + /// + /// All of the versions are prerelease so there are no search documents for "stable" search filters. There + /// two search documents to be updated, one for and one for + /// . The latest version for each of these two + /// documents is different. + var search = new List>(); + var searchFilters = new List(); + foreach (var searchFilter in DocumentUtilities.AllSearchFilters) + { + // Determine if there is a document for this ID and search filter. + if (versionLists.GetLatestVersionInfoOrNull(searchFilter) == null) + { + continue; + } + + var document = buildDocument(searchFilter); + var indexAction = IndexAction.Merge(document); + search.Add(indexAction); + searchFilters.Add(searchFilter); + } + + _logger.LogInformation( + "Package ID {PackageId} has {Count} search document changes for search filters: {SearchFilters}", + packageId, + searchFilters.Count, + searchFilters); + + // No changes are made to the hijack index. + var hijack = new List>(); + + // We never make any change to the version list but still want to push it back to storage. This will give + // us an etag mismatch if the version list has changed. This is good because if the version list has + // changed it's possible there is another search document that we have to update. If we didn't do this, + // then a race condition could occur where one of the search documents for an ID would not get an update. + var newVersionListDataResult = versionListDataResult; + + return new IndexActions( + search, + hijack, + newVersionListDataResult); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/AuxiliaryData.cs b/src/NuGet.Services.AzureSearch/SearchService/AuxiliaryData.cs new file mode 100644 index 000000000..f65f4732e --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/AuxiliaryData.cs @@ -0,0 +1,60 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using NuGet.Services.AzureSearch.AuxiliaryFiles; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class AuxiliaryData : IAuxiliaryData + { + public AuxiliaryData( + DateTimeOffset loaded, + AuxiliaryFileResult downloads, + AuxiliaryFileResult> verifiedPackages, + AuxiliaryFileResult popularityTransfers) + { + Downloads = downloads ?? throw new ArgumentNullException(nameof(downloads)); + VerifiedPackages = verifiedPackages ?? throw new ArgumentNullException(nameof(verifiedPackages)); + PopularityTransfers = popularityTransfers ?? throw new ArgumentNullException(nameof(popularityTransfers)); + + Metadata = new AuxiliaryFilesMetadata( + loaded, + Downloads.Metadata, + VerifiedPackages.Metadata, + PopularityTransfers.Metadata); + } + + internal AuxiliaryFileResult Downloads { get; } + internal AuxiliaryFileResult> VerifiedPackages { get; } + internal AuxiliaryFileResult PopularityTransfers { get; } + public AuxiliaryFilesMetadata Metadata { get; } + + public bool IsVerified(string id) + { + return VerifiedPackages.Data.Contains(id); + } + + public long GetTotalDownloadCount(string id) + { + return Downloads.Data.GetDownloadCount(id); + } + + public long GetDownloadCount(string id, string normalizedVersion) + { + return Downloads.Data.GetDownloadCount(id, normalizedVersion); + } + + public string[] GetPopularityTransfers(string id) + { + if (PopularityTransfers.Data.TryGetValue(id, out var result)) + { + return result.ToArray(); + } + + return Array.Empty(); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/AuxiliaryDataCache.cs b/src/NuGet.Services.AzureSearch/SearchService/AuxiliaryDataCache.cs new file mode 100644 index 000000000..1c5b88433 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/AuxiliaryDataCache.cs @@ -0,0 +1,167 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class AuxiliaryDataCache : IAuxiliaryDataCache + { + private readonly SemaphoreSlim _lock = new SemaphoreSlim(1); + private readonly IDownloadDataClient _downloadDataClient; + private readonly IVerifiedPackagesDataClient _verifiedPackagesDataClient; + private readonly IPopularityTransferDataClient _popularityTransferDataClient; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + private readonly StringCache _stringCache; + private AuxiliaryData _data; + + public AuxiliaryDataCache( + IDownloadDataClient downloadDataClient, + IVerifiedPackagesDataClient verifiedPackagesDataClient, + IPopularityTransferDataClient popularityTransferDataClient, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _downloadDataClient = downloadDataClient ?? throw new ArgumentNullException(nameof(downloadDataClient)); + _verifiedPackagesDataClient = verifiedPackagesDataClient ?? throw new ArgumentNullException(nameof(verifiedPackagesDataClient)); + _popularityTransferDataClient = popularityTransferDataClient ?? throw new ArgumentNullException(nameof(popularityTransferDataClient)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _stringCache = new StringCache(); + } + + public bool Initialized => _data != null; + + public async Task EnsureInitializedAsync() + { + if (!Initialized) + { + await LoadAsync(Timeout.InfiniteTimeSpan, shouldReload: false, token: CancellationToken.None); + } + } + + public async Task TryLoadAsync(CancellationToken token) + { + await LoadAsync(TimeSpan.Zero, shouldReload: true, token: token); + } + + private async Task LoadAsync(TimeSpan timeout, bool shouldReload, CancellationToken token) + { + var acquired = false; + try + { + acquired = await _lock.WaitAsync(timeout, token); + if (!acquired) + { + _logger.LogInformation("Another thread is already reloading the auxiliary data."); + } + else + { + if (!shouldReload && Initialized) + { + return; + } + + _logger.LogInformation("Starting the reload of auxiliary data."); + + var stopwatch = Stopwatch.StartNew(); + + // Load the auxiliary files in parallel. + const string downloadsName = nameof(_data.Downloads); + const string verifiedPackagesName = nameof(_data.VerifiedPackages); + const string popularityTransfersName = nameof(_data.PopularityTransfers); + var downloadsTask = LoadAsync(_data?.Downloads, _downloadDataClient.ReadLatestIndexedAsync); + var verifiedPackagesTask = LoadAsync(_data?.VerifiedPackages, _verifiedPackagesDataClient.ReadLatestAsync); + var popularityTransfersTask = LoadAsync(_data?.PopularityTransfers, _popularityTransferDataClient.ReadLatestIndexedAsync); + await Task.WhenAll(downloadsTask, verifiedPackagesTask); + var downloads = await downloadsTask; + var verifiedPackages = await verifiedPackagesTask; + var popularityTransfers = await popularityTransfersTask; + + // Keep track of what was actually reloaded and what didn't change. + var reloadedNames = new List(); + var notModifiedNames = new List(); + (ReferenceEquals(_data?.Downloads, downloads) ? notModifiedNames : reloadedNames).Add(downloadsName); + (ReferenceEquals(_data?.VerifiedPackages, verifiedPackages) ? notModifiedNames : reloadedNames).Add(verifiedPackagesName); + (ReferenceEquals(_data?.PopularityTransfers, popularityTransfers) ? notModifiedNames : reloadedNames).Add(popularityTransfersName); + + // Reference assignment is atomic, so this is what makes the data available for readers. + _data = new AuxiliaryData( + DateTimeOffset.UtcNow, + downloads, + verifiedPackages, + popularityTransfers); + + // Track the counts regarding the string cache status. + _telemetryService.TrackAuxiliaryFilesStringCache( + _stringCache.StringCount, + _stringCache.CharCount, + _stringCache.RequestCount, + _stringCache.HitCount); + _stringCache.ResetCounts(); + + stopwatch.Stop(); + _telemetryService.TrackAuxiliaryFilesReload(reloadedNames, notModifiedNames, stopwatch.Elapsed); + _logger.LogInformation( + "Done reloading auxiliary data. Took {Duration}. Reloaded: {Reloaded}. Not modified: {NotModified}", + stopwatch.Elapsed, + reloadedNames, + notModifiedNames); + } + } + finally + { + if (acquired) + { + _lock.Release(); + } + } + } + + private async Task> LoadAsync( + AuxiliaryFileResult previousResult, + Func>> getResult) where T : class + { + await Task.Yield(); + + IAccessCondition accessCondition; + if (previousResult == null) + { + accessCondition = AccessConditionWrapper.GenerateEmptyCondition(); + } + else + { + accessCondition = AccessConditionWrapper.GenerateIfNoneMatchCondition(previousResult.Metadata.ETag); + } + + var newResult = await getResult(accessCondition, _stringCache); + if (newResult.Modified) + { + return newResult; + } + else + { + return previousResult; + } + } + + public IAuxiliaryData Get() + { + if (_data == null) + { + throw new InvalidOperationException( + $"The auxiliary data has not been loaded yet. Call {nameof(LoadAsync)}."); + } + + return _data; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/AuxiliaryFileReloader.cs b/src/NuGet.Services.AzureSearch/SearchService/AuxiliaryFileReloader.cs new file mode 100644 index 000000000..a3948fe8f --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/AuxiliaryFileReloader.cs @@ -0,0 +1,63 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Services.AzureSearch.Wrappers; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class AuxiliaryFileReloader : IAuxiliaryFileReloader + { + private readonly IAuxiliaryDataCache _cache; + private readonly ISystemTime _systemTime; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public AuxiliaryFileReloader( + IAuxiliaryDataCache cache, + ISystemTime systemTime, + IOptionsSnapshot options, + ILogger logger) + { + _cache = cache ?? throw new ArgumentNullException(nameof(cache)); + _systemTime = systemTime ?? throw new ArgumentNullException(nameof(systemTime)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ReloadContinuouslyAsync(CancellationToken token) + { + while (!token.IsCancellationRequested) + { + _logger.LogInformation("Trying to reload the auxiliary data."); + + TimeSpan delay; + try + { + await _cache.TryLoadAsync(token); + delay = _options.Value.AuxiliaryDataReloadFrequency; + } + catch (Exception ex) + { + _logger.LogError(0, ex, "An exception was thrown while reloading the auxiliary data."); + + delay = _options.Value.AuxiliaryDataReloadFailureRetryFrequency; + } + + if (token.IsCancellationRequested) + { + return; + } + + _logger.LogInformation( + "Waiting {Duration} before attempting to reload the auxiliary data again.", + delay); + await _systemTime.Delay(delay, token); + } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/AzureSearchService.cs b/src/NuGet.Services.AzureSearch/SearchService/AzureSearchService.cs new file mode 100644 index 000000000..519f86ef9 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/AzureSearchService.cs @@ -0,0 +1,254 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Threading.Tasks; +using NuGet.Services.AzureSearch.Wrappers; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class AzureSearchService : ISearchService + { + private readonly IIndexOperationBuilder _operationBuilder; + private readonly ISearchIndexClientWrapper _searchIndex; + private readonly ISearchIndexClientWrapper _hijackIndex; + private readonly ISearchResponseBuilder _responseBuilder; + private readonly IAzureSearchTelemetryService _telemetryService; + + public AzureSearchService( + IIndexOperationBuilder operationBuilder, + ISearchIndexClientWrapper searchIndex, + ISearchIndexClientWrapper hijackIndex, + ISearchResponseBuilder responseBuilder, + IAzureSearchTelemetryService telemetryService) + { + _operationBuilder = operationBuilder ?? throw new ArgumentNullException(nameof(operationBuilder)); + _searchIndex = searchIndex ?? throw new ArgumentNullException(nameof(searchIndex)); + _hijackIndex = hijackIndex ?? throw new ArgumentNullException(nameof(hijackIndex)); + _responseBuilder = responseBuilder ?? throw new ArgumentNullException(nameof(responseBuilder)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + } + + public async Task V2SearchAsync(V2SearchRequest request) + { + if (request.IgnoreFilter) + { + if (request.PackageType != null && request.PackageType.Any()) + { + throw new InvalidSearchRequestException("Can't apply the packageType filter on the Hijack index"); + } + else if (request.SortBy == V2SortBy.TotalDownloadsAsc || request.SortBy == V2SortBy.TotalDownloadsDesc) + { + throw new InvalidSearchRequestException("Can't sortBy downloads on the Hijack index"); + } + + return await UseHijackIndexAsync(request); + } + else + { + return await UseSearchIndexAsync(request); + } + } + + public async Task V3SearchAsync(V3SearchRequest request) + { + var operation = _operationBuilder.V3Search(request); + + V3SearchResponse output; + switch (operation.Type) + { + case IndexOperationType.Get: + var documentResult = await Measure.DurationWithValueAsync( + () => _searchIndex.Documents.GetOrNullAsync(operation.DocumentKey)); + + output = _responseBuilder.V3FromSearchDocument( + request, + operation.DocumentKey, + documentResult.Value, + documentResult.Duration); + + _telemetryService.TrackV3GetDocument(documentResult.Duration); + break; + + case IndexOperationType.Search: + var result = await Measure.DurationWithValueAsync(() => _searchIndex.Documents.SearchAsync( + operation.SearchText, + operation.SearchParameters)); + + output = _responseBuilder.V3FromSearch( + request, + operation.SearchText, + operation.SearchParameters, + result.Value, + result.Duration); + + _telemetryService.TrackV3SearchQuery(result.Duration); + break; + + case IndexOperationType.Empty: + output = _responseBuilder.EmptyV3(request); + break; + + default: + throw UnsupportedOperation(operation); + } + + return output; + } + + public async Task AutocompleteAsync(AutocompleteRequest request) + { + var operation = _operationBuilder.Autocomplete(request); + + AutocompleteResponse output; + switch (operation.Type) + { + case IndexOperationType.Search: + var result = await Measure.DurationWithValueAsync(() => _searchIndex.Documents.SearchAsync( + operation.SearchText, + operation.SearchParameters)); + + output = _responseBuilder.AutocompleteFromSearch( + request, + operation.SearchText, + operation.SearchParameters, + result.Value, + result.Duration); + + _telemetryService.TrackAutocompleteQuery(result.Duration); + break; + + case IndexOperationType.Empty: + output = _responseBuilder.EmptyAutocomplete(request); + break; + + default: + throw UnsupportedOperation(operation); + } + + return output; + } + + private async Task UseHijackIndexAsync(V2SearchRequest request) + { + var operation = _operationBuilder.V2SearchWithHijackIndex(request); + + V2SearchResponse output; + switch (operation.Type) + { + case IndexOperationType.Get: + var documentResult = await Measure.DurationWithValueAsync( + () => _hijackIndex.Documents.GetOrNullAsync(operation.DocumentKey)); + + // If the request is excluding SemVer 2.0.0 packages and the document is SemVer 2.0.0, filter it + // out. The must be done after fetching the document because some SemVer 2.0.0 packages are + // SemVer 2.0.0 because of a dependency version or because of build metadata (e.g. 1.0.0+metadata). + // Neither of these reasons is apparent from the request. Build metadata is not used for comparison + // so if someone searchs for "version:1.0.0+foo" and the actual package version is "1.0.0" or + // "1.0.0+bar" the document will still be returned. + // + // A request looking for a specific package version that is SemVer 2.0.0 due to dots in the + // prerelease label (e.g. 1.0.0-beta.1) could no-op if the request is not including SemVer 2.0.0 + // but that's not worth the effort since we can catch that case after fetching the document anyway. + // It's more consistent with the search operation path to make the SemVer 2.0.0 filtering decision + // solely based on the document data. + // + // Note that the prerelease filter is ignored here by design. This is legacy behavior from the + // previous search implementation. + var document = documentResult.Value; + if (document != null + && !request.IncludeSemVer2 + && document.SemVerLevel == SemVerLevelKey.SemVer2) + { + document = null; + } + + output = _responseBuilder.V2FromHijackDocument( + request, + operation.DocumentKey, + document, + documentResult.Duration); + + _telemetryService.TrackV2GetDocumentWithHijackIndex(documentResult.Duration); + break; + + case IndexOperationType.Search: + var result = await Measure.DurationWithValueAsync(() => _hijackIndex.Documents.SearchAsync( + operation.SearchText, + operation.SearchParameters)); + + output = _responseBuilder.V2FromHijack( + request, + operation.SearchText, + operation.SearchParameters, + result.Value, + result.Duration); + + _telemetryService.TrackV2SearchQueryWithHijackIndex(result.Duration); + break; + + case IndexOperationType.Empty: + output = _responseBuilder.EmptyV2(request); + break; + + default: + throw UnsupportedOperation(operation); + } + + return output; + } + + private async Task UseSearchIndexAsync(V2SearchRequest request) + { + var operation = _operationBuilder.V2SearchWithSearchIndex(request); + + V2SearchResponse output; + switch (operation.Type) + { + case IndexOperationType.Get: + var documentResult = await Measure.DurationWithValueAsync( + () => _searchIndex.Documents.GetOrNullAsync(operation.DocumentKey)); + + output = _responseBuilder.V2FromSearchDocument( + request, + operation.DocumentKey, + documentResult.Value, + documentResult.Duration); + + _telemetryService.TrackV2GetDocumentWithSearchIndex(documentResult.Duration); + break; + + case IndexOperationType.Search: + var result = await Measure.DurationWithValueAsync(() => _searchIndex.Documents.SearchAsync( + operation.SearchText, + operation.SearchParameters)); + + output = _responseBuilder.V2FromSearch( + request, + operation.SearchText, + operation.SearchParameters, + result.Value, + result.Duration); + + _telemetryService.TrackV2SearchQueryWithSearchIndex(result.Duration); + break; + + case IndexOperationType.Empty: + output = _responseBuilder.EmptyV2(request); + break; + + default: + throw UnsupportedOperation(operation); + } + + return output; + } + + private static NotImplementedException UnsupportedOperation(IndexOperation operation) + { + return new NotImplementedException($"The operation type {operation.Type} is not supported."); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/AzureSearchTextBuilder.cs b/src/NuGet.Services.AzureSearch/SearchService/AzureSearchTextBuilder.cs new file mode 100644 index 000000000..f650f1a7c --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/AzureSearchTextBuilder.cs @@ -0,0 +1,309 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public partial class SearchTextBuilder + { + /// + /// Used to build Azure Search Service queries. + /// + /// + /// This generates Azure Search queries that use the Lucene query syntax. + /// See: https://docs.microsoft.com/en-us/azure/search/query-lucene-syntax + /// + /// Given the query "fieldA:value1 value2": + /// + /// * "value1" is a field-scoped term + /// * "value2" is an unscoped term + /// + private class AzureSearchTextBuilder + { + /// + /// Azure Search Queries must have less than 1024 clauses. + /// See: https://docs.microsoft.com/en-us/azure/search/query-lucene-syntax#bkmk_querysizelimits + /// + private const int MaxClauses = 1024; + + /// + /// Terms in Azure Search Queries must be less than 32KB. + /// See: https://docs.microsoft.com/en-us/azure/search/query-lucene-syntax#bkmk_querysizelimits + /// + private const int MaxTermSizeBytes = 32 * 1024; + + /// + /// These characters have special meaning in Azure Search and must be escaped if in user input. + /// See: https://docs.microsoft.com/en-us/azure/search/query-lucene-syntax#escaping-special-characters + /// + private static readonly HashSet SpecialCharacters = new HashSet + { + '+', '-', '&', '|', '!', '(', ')', '{', '}', '[', ']', '^', '"', '~', '*', '?', ':', '\\', '/', + }; + + private readonly StringBuilder _result; + private int _clauses; + + public AzureSearchTextBuilder() + { + _result = new StringBuilder(); + _clauses = 0; + } + + /// + /// Append search terms to the query. These terms may match any field. + /// + /// The terms to append to the search query. + public void AppendTerms(IReadOnlyList terms) + { + ValidateAdditionalClausesOrThrow(terms.Count); + ValidateTermsOrThrow(terms); + + AppendSpaceIfNotEmpty(); + + for (var i = 0; i < terms.Count; i++) + { + if (i > 0) + { + _result.Append(' '); + } + + AppendEscapedString(terms[i], quoteWhiteSpace: true); + } + } + + /// + /// Append a clause to boost results that match all terms. + /// + /// All terms that must be matched. + /// The boost for results that match all terms. + public void AppendBoostIfMatchAllTerms(IReadOnlyList terms, float boost) + { + // We will generate a clause for each term and a clause to OR terms together. + ValidateAdditionalClausesOrThrow(terms.Count + 1); + ValidateTermsOrThrow(terms); + + AppendSpaceIfNotEmpty(); + + _result.Append('('); + + for (var i = 0; i < terms.Count; i++) + { + if (i > 0) + { + _result.Append(' '); + } + + _result.Append('+'); + AppendEscapedString(terms[i], quoteWhiteSpace: true); + } + + _result.Append(")^"); + _result.Append(boost); + } + + /// + /// Append a clause to boost an exact matched package ID. + /// + /// The package ID that must can be matched. + /// The boost for the document with the matching package ID. + public void AppendExactMatchPackageIdBoost(string packageId, float boost) + { + ValidateAdditionalClausesOrThrow(1); + ValidateTermsOrThrow(new[] { packageId }); + + AppendSpaceIfNotEmpty(); + + _result.Append(IndexFields.PackageId); + _result.Append(":"); + AppendEscapedString(packageId, quoteWhiteSpace: true); + _result.Append("^"); + _result.Append(boost); + } + + /// + /// Append a term to the query that is scoped to a specified field. This generates + /// queries like "field:value". Unlike , this supports + /// prefix matching. + /// + /// The field that should contain this term. + /// The term to search. + /// Whether search results MUST match this term. + /// Whether prefix matches are allowed. + /// The boost to results that match this term. + public void AppendScopedTerm( + string fieldName, + string term, + bool required = false, + bool prefixSearch = false, + double boost = 1.0) + { + // We will generate a single clause. + ValidateAdditionalClausesOrThrow(1); + ValidateTermOrThrow(term); + + AppendSpaceIfNotEmpty(); + + if (required) + { + _result.Append('+'); + } + + _result.Append(fieldName); + _result.Append(':'); + + // Don't escape whitespace with quotes if this is prefix matching. + AppendEscapedString(term.Trim(), quoteWhiteSpace: !prefixSearch); + + if (prefixSearch) + { + _result.Append('*'); + } + + if (boost > 1) + { + _result.Append("^"); + _result.Append(boost); + } + } + + /// + /// Append search terms to the query that are scoped to a specified field. + /// This generates queries like "field:(value1 value2)". Unlike + /// , this doesn't support prefix matches. + /// + /// The field that should match the terms. + /// The terms to search + /// Whether search results MUST match these terms. + public void AppendScopedTerms( + string fieldName, + IReadOnlyList terms, + bool required = false) + { + // We will generate a clause for each term and a clause to OR terms together. + ValidateAdditionalClausesOrThrow(terms.Count + 1); + ValidateTermsOrThrow(terms); + + AppendSpaceIfNotEmpty(); + + if (required) + { + _result.Append('+'); + } + + _result.Append(fieldName); + _result.Append(":("); + + for (var i = 0; i < terms.Count; i++) + { + if (i > 0) + { + _result.Append(' '); + } + + AppendEscapedString(terms[i].Trim(), quoteWhiteSpace: true); + } + + _result.Append(')'); + } + + /// + /// Build the Azure Search Query string. + /// + /// The Azure Search Query string. + public override string ToString() + { + return _result.ToString(); + } + + private void AppendSpaceIfNotEmpty() + { + if (_result.Length > 0) + { + _result.Append(' '); + } + } + + /// + /// Escapes characters that are special for Azure Search so that the input + /// results in a single search term. + /// + /// The input to escape. + /// + /// If true, the input will be wrapped with quotes if it contains whitespace. + /// If false, the input's whitespace will be escaped with a backslash. + /// + private void AppendEscapedString(string input, bool quoteWhiteSpace) + { + var originalLength = _result.Length; + + // Input containing whitespace must be escaped. If quoteWhiteSpace is true, we + // will wrap the input with quotes. Otherwise, we will escape whitespace characters + // with backslashes. + var wrapWithQuotes = quoteWhiteSpace && input.Any(char.IsWhiteSpace); + if (wrapWithQuotes) + { + _result.Append('"'); + } + + for (var i = 0; i < input.Length; i++) + { + var c = input[i]; + if (SpecialCharacters.Contains(c) || (!quoteWhiteSpace && char.IsWhiteSpace(c))) + { + if (originalLength == _result.Length) + { + _result.Append(input.Substring(0, i)); + } + + _result.Append('\\'); + _result.Append(c); + } + else if (_result.Length != originalLength) + { + _result.Append(c); + } + } + + if (wrapWithQuotes) + { + _result.Append('"'); + } + + if (_result.Length == originalLength) + { + _result.Append(input); + } + } + + private void ValidateAdditionalClausesOrThrow(int additionalClauses) + { + if ((_clauses + additionalClauses) > MaxClauses) + { + throw new InvalidSearchRequestException($"A query can only have up to {MaxClauses} clauses."); + } + + _clauses += additionalClauses; + } + + private void ValidateTermsOrThrow(IReadOnlyList terms) + { + foreach (var term in terms) + { + ValidateTermOrThrow(term); + } + } + + private void ValidateTermOrThrow(string term) + { + if (Encoding.Unicode.GetByteCount(term) > MaxTermSizeBytes) + { + throw new InvalidSearchRequestException($"Query terms cannot exceed {MaxTermSizeBytes} bytes."); + } + } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/IAuxiliaryData.cs b/src/NuGet.Services.AzureSearch/SearchService/IAuxiliaryData.cs new file mode 100644 index 000000000..bd70c52b1 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/IAuxiliaryData.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.SearchService +{ + public interface IAuxiliaryData + { + AuxiliaryFilesMetadata Metadata { get; } + long GetDownloadCount(string id, string normalizedVersion); + long GetTotalDownloadCount(string id); + bool IsVerified(string id); + string[] GetPopularityTransfers(string id); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/SearchService/IAuxiliaryDataCache.cs b/src/NuGet.Services.AzureSearch/SearchService/IAuxiliaryDataCache.cs new file mode 100644 index 000000000..c2b50c1ed --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/IAuxiliaryDataCache.cs @@ -0,0 +1,40 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public interface IAuxiliaryDataCache + { + /// + /// Returns true if there is auxiliary data available. False, otherwise. If there is data available, it can be + /// retrieved using . + /// + bool Initialized { get; } + + /// + /// Returns the cached loaded auxiliary data. + /// + /// + /// Thrown if there is not data available. should be called if this is + /// thrown. can be used to check whether data is available. + /// + IAuxiliaryData Get(); + + /// + /// Load the latest version of the auxiliary data if it is not already loaded. If the data is already being + /// loaded by another caller, this method waits until that other reload finishes and ensures that the data was + /// loaded. If the other caller successfully loaded the auxiliary data, this method will no-op. + /// + Task EnsureInitializedAsync(); + + /// + /// Tries to load the latest version of the auxiliary data. If the data is already being loaded by another + /// caller, this method does not reload the data but does wait until some data is available. + /// + Task TryLoadAsync(CancellationToken token); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/SearchService/IAuxiliaryFileReloader.cs b/src/NuGet.Services.AzureSearch/SearchService/IAuxiliaryFileReloader.cs new file mode 100644 index 000000000..b39357626 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/IAuxiliaryFileReloader.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public interface IAuxiliaryFileReloader + { + Task ReloadContinuouslyAsync(CancellationToken token); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/SearchService/IIndexOperationBuilder.cs b/src/NuGet.Services.AzureSearch/SearchService/IIndexOperationBuilder.cs new file mode 100644 index 000000000..40e75048b --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/IIndexOperationBuilder.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.SearchService +{ + /// + /// This interface encapsulates the selection of Azure Search operation as well as the generation of parameters for + /// the selected operation. Query optimizations such as looking up a document by key instead of doing a full Lucene + /// query are decided here. + /// + public interface IIndexOperationBuilder + { + IndexOperation Autocomplete(AutocompleteRequest request); + IndexOperation V2SearchWithHijackIndex(V2SearchRequest request); + IndexOperation V2SearchWithSearchIndex(V2SearchRequest request); + IndexOperation V3Search(V3SearchRequest request); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/SearchService/ISearchParametersBuilder.cs b/src/NuGet.Services.AzureSearch/SearchService/ISearchParametersBuilder.cs new file mode 100644 index 000000000..dae65414f --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/ISearchParametersBuilder.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public interface ISearchParametersBuilder + { + SearchParameters LastCommitTimestamp(); + SearchParameters V2Search(V2SearchRequest request, bool isDefaultSearch); + SearchParameters V3Search(V3SearchRequest request, bool isDefaultSearch); + SearchParameters Autocomplete(AutocompleteRequest request, bool isDefaultSearch); + SearchFilters GetSearchFilters(SearchRequest request); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/SearchService/ISearchResponseBuilder.cs b/src/NuGet.Services.AzureSearch/SearchService/ISearchResponseBuilder.cs new file mode 100644 index 000000000..60e31b4a8 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/ISearchResponseBuilder.cs @@ -0,0 +1,54 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public interface ISearchResponseBuilder + { + V2SearchResponse V2FromHijack( + V2SearchRequest request, + string text, + SearchParameters parameters, + DocumentSearchResult result, + TimeSpan duration); + V2SearchResponse V2FromSearch( + V2SearchRequest request, + string text, + SearchParameters parameters, + DocumentSearchResult result, + TimeSpan duration); + V2SearchResponse V2FromHijackDocument( + V2SearchRequest request, + string documentKey, + HijackDocument.Full document, + TimeSpan duration); + V3SearchResponse V3FromSearch( + V3SearchRequest request, + string text, + SearchParameters parameters, + DocumentSearchResult result, + TimeSpan duration); + V2SearchResponse V2FromSearchDocument( + V2SearchRequest request, + string documentKey, + SearchDocument.Full document, + TimeSpan duration); + V3SearchResponse V3FromSearchDocument( + V3SearchRequest request, + string documentKey, + SearchDocument.Full document, + TimeSpan duration); + AutocompleteResponse AutocompleteFromSearch( + AutocompleteRequest request, + string text, + SearchParameters parameters, + DocumentSearchResult result, + TimeSpan duration); + V2SearchResponse EmptyV2(V2SearchRequest request); + V3SearchResponse EmptyV3(V3SearchRequest request); + AutocompleteResponse EmptyAutocomplete(AutocompleteRequest request); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/SearchService/ISearchService.cs b/src/NuGet.Services.AzureSearch/SearchService/ISearchService.cs new file mode 100644 index 000000000..e97ded873 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/ISearchService.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public interface ISearchService + { + /// + /// Perform a V2 search query. + /// + /// The V2 search request. + /// The V2 search response. + /// Thrown if the request is invalid. + Task V2SearchAsync(V2SearchRequest request); + + /// + /// Perform a V3 search query. + /// + /// The V3 search request. + /// The V3 search response. + /// Thrown if the request is invalid. + Task V3SearchAsync(V3SearchRequest request); + + /// + /// Perform an autocomplete query. + /// + /// The autocomplete request. + /// The autocomplete response. + /// Thrown if the request is invalid. + Task AutocompleteAsync(AutocompleteRequest request); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/SearchService/ISearchStatusService.cs b/src/NuGet.Services.AzureSearch/SearchService/ISearchStatusService.cs new file mode 100644 index 000000000..93683fc21 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/ISearchStatusService.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public interface ISearchStatusService + { + Task GetStatusAsync(SearchStatusOptions options, Assembly assemblyForMetadata); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/SearchService/ISearchTextBuilder.cs b/src/NuGet.Services.AzureSearch/SearchService/ISearchTextBuilder.cs new file mode 100644 index 000000000..b62196d76 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/ISearchTextBuilder.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.SearchService +{ + /// + /// Generates Azure Search Query using Lucene query syntax. + /// See: https://docs.microsoft.com/en-us/azure/search/query-lucene-syntax + /// + public interface ISearchTextBuilder + { + /// + /// Map a V2 search request to Azure Search. + /// + /// The V2 search request. + /// The Azure Search query. + /// Thrown on invalid search requests. + ParsedQuery ParseV2Search(V2SearchRequest request); + + /// + /// Map a V3 search request to Azure Search. + /// + /// The V3 search request. + /// The Azure Search query. + /// Thrown on invalid search requests. + ParsedQuery ParseV3Search(V3SearchRequest request); + + /// + /// Build a parsed query into search text. + /// + /// The parsed query. + /// The Lucene search text to pass to Azure Search. + string Build(ParsedQuery parsed); + + /// + /// Map an autocomplete request to Azure Search. + /// + /// The autocomplete request. + /// The Azure Search query. + /// Thrown on invalid autocomplete requests. + string Autocomplete(AutocompleteRequest request); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/SearchService/ISecretRefresher.cs b/src/NuGet.Services.AzureSearch/SearchService/ISecretRefresher.cs new file mode 100644 index 000000000..4403501f6 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/ISecretRefresher.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch.SearchService +{ + /// + /// Used to update the secrets in the background periodically. + /// + public interface ISecretRefresher + { + DateTimeOffset LastRefresh { get; } + Task RefreshContinuouslyAsync(CancellationToken token); + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/IndexFields.cs b/src/NuGet.Services.AzureSearch/SearchService/IndexFields.cs new file mode 100644 index 000000000..933da37cb --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/IndexFields.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json.Serialization; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public static class IndexFields + { + private static readonly NamingStrategy CamelCaseNamingStrategy = new CamelCaseNamingStrategy(); + + private static string Name(string input) + { + return CamelCaseNamingStrategy.GetPropertyName(input, hasSpecifiedName: false); + } + + public static readonly string Authors = Name(nameof(BaseMetadataDocument.Authors)); + public static readonly string Created = Name(nameof(BaseMetadataDocument.Created)); + public static readonly string Description = Name(nameof(BaseMetadataDocument.Description)); + public static readonly string LastCommitTimestamp = Name(nameof(BaseMetadataDocument.LastCommitTimestamp)); + public static readonly string LastEdited = Name(nameof(BaseMetadataDocument.LastEdited)); + public static readonly string NormalizedVersion = Name(nameof(BaseMetadataDocument.NormalizedVersion)); + public static readonly string PackageId = Name(nameof(BaseMetadataDocument.PackageId)); + public static readonly string Published = Name(nameof(BaseMetadataDocument.Published)); + public static readonly string SemVerLevel = Name(nameof(BaseMetadataDocument.SemVerLevel)); + public static readonly string SortableTitle = Name(nameof(BaseMetadataDocument.SortableTitle)); + public static readonly string Summary = Name(nameof(BaseMetadataDocument.Summary)); + public static readonly string Tags = Name(nameof(BaseMetadataDocument.Tags)); + public static readonly string Title = Name(nameof(BaseMetadataDocument.Title)); + public static readonly string TokenizedPackageId = Name(nameof(BaseMetadataDocument.TokenizedPackageId)); + + public static class Search + { + public static readonly string DownloadScore = Name(nameof(SearchDocument.Full.DownloadScore)); + public static readonly string FilterablePackageTypes = Name(nameof(SearchDocument.UpdateLatest.FilterablePackageTypes)); + public static readonly string IsExcludedByDefault = Name(nameof(SearchDocument.Full.IsExcludedByDefault)); + public static readonly string Owners = Name(nameof(SearchDocument.Full.Owners)); + public static readonly string SearchFilters = Name(nameof(SearchDocument.UpdateLatest.SearchFilters)); + public static readonly string TotalDownloadCount = Name(nameof(SearchDocument.Full.TotalDownloadCount)); + public static readonly string Versions = Name(nameof(SearchDocument.UpdateLatest.Versions)); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/IndexOperation.cs b/src/NuGet.Services.AzureSearch/SearchService/IndexOperation.cs new file mode 100644 index 000000000..881bce061 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/IndexOperation.cs @@ -0,0 +1,72 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class IndexOperation + { + private IndexOperation( + IndexOperationType type, + string documentKey, + string searchText, + SearchParameters searchParameters) + { + Type = type; + DocumentKey = documentKey; + SearchText = searchText; + SearchParameters = searchParameters; + } + + /// + /// The type of index operation. This is used to determine which other properties are applicable. + /// + public IndexOperationType Type { get; } + + /// + /// The key to look up an Azure Search document with. + /// Used when is . + /// + public string DocumentKey { get; } + + /// + /// The text to use for a search query. + /// Used when is . + /// + public string SearchText { get; } + + /// + /// The parameters to use for an Azure Search query. + /// Used when is . + /// + public SearchParameters SearchParameters { get; } + + public static IndexOperation Get(string documentKey) + { + return new IndexOperation( + IndexOperationType.Get, + documentKey, + searchText: null, + searchParameters: null); + } + + public static IndexOperation Search(string text, SearchParameters parameters) + { + return new IndexOperation( + IndexOperationType.Search, + documentKey: null, + searchText: text, + searchParameters: parameters); + } + + public static IndexOperation Empty() + { + return new IndexOperation( + IndexOperationType.Empty, + documentKey: null, + searchText: null, + searchParameters: null); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/IndexOperationBuilder.cs b/src/NuGet.Services.AzureSearch/SearchService/IndexOperationBuilder.cs new file mode 100644 index 000000000..c6430e508 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/IndexOperationBuilder.cs @@ -0,0 +1,199 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using NuGet.Packaging; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class IndexOperationBuilder : IIndexOperationBuilder + { + /// + /// Azure Search can only skip up to 100,000 documents. + /// https://docs.microsoft.com/en-us/rest/api/searchservice/search-documents#skip-optional + /// + private const int MaximumSkip = 100000; + + private readonly ISearchTextBuilder _textBuilder; + private readonly ISearchParametersBuilder _parametersBuilder; + + public IndexOperationBuilder( + ISearchTextBuilder textBuilder, + ISearchParametersBuilder parametersBuilder) + { + _textBuilder = textBuilder ?? throw new ArgumentNullException(nameof(textBuilder)); + _parametersBuilder = parametersBuilder ?? throw new ArgumentNullException(nameof(parametersBuilder)); + } + + public IndexOperation V3Search(V3SearchRequest request) + { + if (HasInvalidParameters(request, request.PackageType)) + { + return IndexOperation.Empty(); + } + + var parsed = _textBuilder.ParseV3Search(request); + + IndexOperation indexOperation; + if (request.PackageType == null + && TryGetSearchDocumentByKey(request, parsed, out indexOperation)) + { + return indexOperation; + } + + var text = _textBuilder.Build(parsed); + var parameters = _parametersBuilder.V3Search(request, IsEmptySearchQuery(text)); + return IndexOperation.Search(text, parameters); + } + + public IndexOperation V2SearchWithSearchIndex(V2SearchRequest request) + { + if (HasInvalidParameters(request, request.PackageType)) + { + return IndexOperation.Empty(); + } + + var parsed = _textBuilder.ParseV2Search(request); + + IndexOperation indexOperation; + if (request.PackageType == null + && TryGetSearchDocumentByKey(request, parsed, out indexOperation)) + { + return indexOperation; + } + + var text = _textBuilder.Build(parsed); + var parameters = _parametersBuilder.V2Search(request, IsEmptySearchQuery(text)); + return IndexOperation.Search(text, parameters); + } + + public IndexOperation V2SearchWithHijackIndex(V2SearchRequest request) + { + if (HasInvalidParameters(request, packageType: null)) + { + return IndexOperation.Empty(); + } + + var parsed = _textBuilder.ParseV2Search(request); + + IndexOperation indexOperation; + if (TryGetHijackDocumentByKey(request, parsed, out indexOperation)) + { + return indexOperation; + } + + var text = _textBuilder.Build(parsed); + var parameters = _parametersBuilder.V2Search(request, IsEmptySearchQuery(text)); + return IndexOperation.Search(text, parameters); + } + + public IndexOperation Autocomplete(AutocompleteRequest request) + { + if (HasInvalidParameters(request, request.PackageType)) + { + return IndexOperation.Empty(); + } + + var text = _textBuilder.Autocomplete(request); + var parameters = _parametersBuilder.Autocomplete(request, IsEmptySearchQuery(text)); + return IndexOperation.Search(text, parameters); + } + + private bool TryGetSearchDocumentByKey( + SearchRequest request, + ParsedQuery parsed, + out IndexOperation indexOperation) + { + if (PagedToFirstItem(request) + && parsed.Grouping.Count == 1 + && TryGetSinglePackageId(parsed, out var packageId)) + { + var searchFilters = _parametersBuilder.GetSearchFilters(request); + var documentKey = DocumentUtilities.GetSearchDocumentKey(packageId, searchFilters); + + indexOperation = IndexOperation.Get(documentKey); + return true; + } + + indexOperation = null; + return false; + } + + private bool TryGetHijackDocumentByKey( + SearchRequest request, + ParsedQuery parsed, + out IndexOperation indexOperation) + { + if (PagedToFirstItem(request) + && parsed.Grouping.Count == 2 + && TryGetSinglePackageId(parsed, out var packageId) + && TryGetSingleVersion(parsed, out var normalizedVersion)) + { + var documentKey = DocumentUtilities.GetHijackDocumentKey(packageId, normalizedVersion); + + indexOperation = IndexOperation.Get(documentKey); + return true; + } + + indexOperation = null; + return false; + } + + private bool TryGetSinglePackageId( + ParsedQuery parsed, + out string packageId) + { + if (parsed.Grouping.TryGetValue(QueryField.PackageId, out var terms) + && terms.Count == 1) + { + packageId = terms.First(); + if (packageId.Length <= PackageIdValidator.MaxPackageIdLength + && PackageIdValidator.IsValidPackageId(packageId)) + { + return true; + } + } + + packageId = null; + return false; + } + + private bool TryGetSingleVersion( + ParsedQuery parsed, + out string normalizedVersion) + { + if (parsed.Grouping.TryGetValue(QueryField.Version, out var terms) + && terms.Count == 1) + { + if (NuGetVersion.TryParse(terms.First(), out var parsedVersion)) + { + normalizedVersion = parsedVersion.ToNormalizedString(); + return true; + } + } + + normalizedVersion = null; + return false; + } + + private static bool HasInvalidParameters(SearchRequest request, string packageType) + { + // Requests with bad parameters yield no results. For the package type case, by specification a package type + // valid characters are the same as a package ID. + return request.Skip > MaximumSkip + || (packageType != null && !PackageIdValidator.IsValidPackageId(packageType)); + } + + private static bool PagedToFirstItem(SearchRequest request) + { + return request.Skip <= 0 && request.Take >= 1; + } + + private static bool IsEmptySearchQuery(string parsedText) + { + return parsedText.Equals(SearchTextBuilder.MatchAllDocumentsQuery); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/IndexOperationType.cs b/src/NuGet.Services.AzureSearch/SearchService/IndexOperationType.cs new file mode 100644 index 000000000..47adb6955 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/IndexOperationType.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.SearchService +{ + public enum IndexOperationType + { + /// + /// The data for the user was fetched using Azure Search's "get document by key" API. The .NET API is called "Get" and + /// "GetAsync" but REST API is called "lookup". + /// https://docs.microsoft.com/en-us/rest/api/searchservice/lookup-document + /// + Get, + + /// + /// The data for the user was fetched using Azure Search's "search documents" API. + /// https://docs.microsoft.com/en-us/rest/api/searchservice/search-documents + /// + Search, + + /// + /// The request should yield an empty response so no Azure Search query is necessary. + /// + Empty, + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/InvalidSearchRequestException.cs b/src/NuGet.Services.AzureSearch/SearchService/InvalidSearchRequestException.cs new file mode 100644 index 000000000..0acece513 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/InvalidSearchRequestException.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch.SearchService +{ + /// + /// This exception is meant to be caught by an exception filter in the web application so that HTTP status code 400 + /// is returned to the user. The message in this exception is meant to become visible to the user. + /// + public class InvalidSearchRequestException : Exception + { + /// + /// Create a new invalid search request exception. + /// + /// The message to display to the user. Must not contain sensitive information. + public InvalidSearchRequestException(string message) + : base(message) + { + } + + /// + /// Create a new invalid search request exception with a provided inner exception for debugging. + /// + /// The message to display to the user. Must not contain sensitive information. + /// This exception will not be shown to the user. + public InvalidSearchRequestException(string message, Exception innerException) + : base(message, innerException) + { + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/AutocompleteContext.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/AutocompleteContext.cs new file mode 100644 index 000000000..b98729799 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/AutocompleteContext.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class AutocompleteContext + { + [JsonProperty("@vocab")] + public string Vocab { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/AutocompleteRequest.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/AutocompleteRequest.cs new file mode 100644 index 000000000..99cc770ea --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/AutocompleteRequest.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.SearchService +{ + /// + /// Source: https://docs.microsoft.com/en-us/nuget/api/search-autocomplete-service-resource#request-parameters + /// + public class AutocompleteRequest : SearchRequest + { + public AutocompleteRequestType Type { get; set; } + public string PackageType { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/AutocompleteRequestType.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/AutocompleteRequestType.cs new file mode 100644 index 000000000..e3bf6cdd4 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/AutocompleteRequestType.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.SearchService +{ + public enum AutocompleteRequestType + { + /// + /// The response's data should list matching package IDs. + /// + PackageIds, + + /// + /// The response should list the package's versions. + /// + PackageVersions, + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/AutocompleteResponse.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/AutocompleteResponse.cs new file mode 100644 index 000000000..7dccd6706 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/AutocompleteResponse.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch.SearchService +{ + /// + /// Source: https://docs.microsoft.com/en-us/nuget/api/search-autocomplete-service-resource#response + /// + public class AutocompleteResponse + { + [JsonProperty("@context")] + public AutocompleteContext Context { get; set; } + + [JsonProperty("totalHits")] + public long TotalHits { get; set; } + + [JsonProperty("data")] + public List Data { get; set; } + + public DebugInformation Debug { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/AuxiliaryFilesMetadata.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/AuxiliaryFilesMetadata.cs new file mode 100644 index 000000000..2f701a647 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/AuxiliaryFilesMetadata.cs @@ -0,0 +1,30 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; +using NuGet.Services.AzureSearch.AuxiliaryFiles; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class AuxiliaryFilesMetadata + { + [JsonConstructor] + public AuxiliaryFilesMetadata( + DateTimeOffset loaded, + AuxiliaryFileMetadata downloads, + AuxiliaryFileMetadata verifiedPackages, + AuxiliaryFileMetadata popularityTransfers) + { + Loaded = loaded; + Downloads = downloads ?? throw new ArgumentNullException(nameof(downloads)); + VerifiedPackages = verifiedPackages ?? throw new ArgumentNullException(nameof(verifiedPackages)); + PopularityTransfers = popularityTransfers ?? throw new ArgumentNullException(nameof(popularityTransfers)); + } + + public DateTimeOffset Loaded { get; } + public AuxiliaryFileMetadata Downloads { get; } + public AuxiliaryFileMetadata VerifiedPackages { get; } + public AuxiliaryFileMetadata PopularityTransfers { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/DebugDocumentResult.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/DebugDocumentResult.cs new file mode 100644 index 000000000..0f53d1385 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/DebugDocumentResult.cs @@ -0,0 +1,10 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class DebugDocumentResult + { + public object Document { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/DebugInformation.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/DebugInformation.cs new file mode 100644 index 000000000..7e593fda2 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/DebugInformation.cs @@ -0,0 +1,85 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class DebugInformation + { + public SearchRequest SearchRequest { get; set; } + public string IndexName { get; set; } + public IndexOperationType IndexOperationType { get; set; } + public string DocumentKey { get; set; } + public SearchParameters SearchParameters { get; set; } + public string SearchText { get; set; } + public object DocumentSearchResult { get; set; } + public TimeSpan? QueryDuration { get; set; } + public AuxiliaryFilesMetadata AuxiliaryFilesMetadata { get; set; } + + public static DebugInformation CreateFromEmptyOrNull(SearchRequest request) + { + if (!request.ShowDebug) + { + return null; + } + + return new DebugInformation + { + SearchRequest = request, + IndexOperationType = IndexOperationType.Empty, + }; + } + + public static DebugInformation CreateFromSearchOrNull( + SearchRequest request, + string indexName, + SearchParameters parameters, + string text, + DocumentSearchResult result, + TimeSpan duration, + AuxiliaryFilesMetadata auxiliaryFilesMetadata) where T : class + { + if (!request.ShowDebug) + { + return null; + } + + return new DebugInformation + { + SearchRequest = request, + IndexName = indexName, + IndexOperationType = IndexOperationType.Search, + SearchParameters = parameters, + SearchText = text, + DocumentSearchResult = result, + QueryDuration = duration, + AuxiliaryFilesMetadata = auxiliaryFilesMetadata, + }; + } + + public static DebugInformation CreateFromGetOrNull( + SearchRequest request, + string indexName, + string documentKey, + TimeSpan duration, + AuxiliaryFilesMetadata auxiliaryFilesMetadata) + { + if (!request.ShowDebug) + { + return null; + } + + return new DebugInformation + { + SearchRequest = request, + IndexName = indexName, + IndexOperationType = IndexOperationType.Get, + DocumentKey = documentKey, + QueryDuration = duration, + AuxiliaryFilesMetadata = auxiliaryFilesMetadata, + }; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/IndexStatus.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/IndexStatus.cs new file mode 100644 index 000000000..0aa96ac83 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/IndexStatus.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class IndexStatus + { + public string Name { get; set; } + public long DocumentCount { get; set; } + public TimeSpan DocumentCountDuration { get; set; } + public TimeSpan WarmQueryDuration { get; set; } + public DateTimeOffset? LastCommitTimestamp { get; set; } + public TimeSpan LastCommitTimestampDuration { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/SearchRequest.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/SearchRequest.cs new file mode 100644 index 000000000..79cd32176 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/SearchRequest.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class SearchRequest + { + public int Skip { get; set; } + public int Take { get; set; } + public bool IncludePrerelease { get; set; } + public bool IncludeSemVer2 { get; set; } + public string Query { get; set; } + public bool ShowDebug { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/SearchStatusResponse.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/SearchStatusResponse.cs new file mode 100644 index 000000000..8ba49a114 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/SearchStatusResponse.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class SearchStatusResponse + { + /// + /// This success boolean indicates whether all dependencies of the search service can be communicated with. + /// Any of the properties on this type, aside from or , can be null + /// if is false. If is true, all properties will be non-null. + /// + public bool Success { get; set; } + public TimeSpan? Duration { get; set; } + public ServerStatus Server { get; set; } + public IndexStatus SearchIndex { get; set; } + public IndexStatus HijackIndex { get; set; } + public AuxiliaryFilesMetadata AuxiliaryFiles { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/ServerInformation.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/ServerInformation.cs new file mode 100644 index 000000000..708eab709 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/ServerInformation.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class ServerStatus + { + public string MachineName { get; set; } + public int ProcessId { get; set; } + public DateTimeOffset ProcessStartTime { get; set; } + public TimeSpan ProcessDuration { get; set; } + public string DeploymentLabel { get; set; } + public string AssemblyCommitId { get; set; } + public string AssemblyInformationalVersion { get; set; } + public string AssemblyBuildDateUtc { get; set; } + public string InstanceId { get; set; } + public DateTimeOffset? LastServiceRefreshTime { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchDependency.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchDependency.cs new file mode 100644 index 000000000..46a99624b --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchDependency.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class V2SearchDependency + { + public string Id { get; set; } + public string VersionSpec { get; set; } + public string TargetFramework { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchPackage.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchPackage.cs new file mode 100644 index 000000000..4cd71494b --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchPackage.cs @@ -0,0 +1,51 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class V2SearchPackage + { + public V2SearchPackageRegistration PackageRegistration { get; set; } + public string Version { get; set; } + public string NormalizedVersion { get; set; } + public string Title { get; set; } + public string Description { get; set; } + public string Summary { get; set; } + public string Authors { get; set; } + public string Copyright { get; set; } + public string Language { get; set; } + public string Tags { get; set; } + public string ReleaseNotes { get; set; } + public string ProjectUrl { get; set; } + public string IconUrl { get; set; } + public bool IsLatestStable { get; set; } + public bool IsLatest { get; set; } + public bool Listed { get; set; } + public DateTimeOffset Created { get; set; } + public DateTimeOffset Published { get; set; } + public DateTimeOffset LastUpdated { get; set; } + public DateTimeOffset? LastEdited { get; set; } + public long DownloadCount { get; set; } + public string FlattenedDependencies { get; set; } + + /// + /// Unused by gallery. + /// + public V2SearchDependency[] Dependencies { get; set; } + + /// + /// Unused by gallery. + /// + public string[] SupportedFrameworks { get; set; } + + public string MinClientVersion { get; set; } + public string Hash { get; set; } + public string HashAlgorithm { get; set; } + public long PackageFileSize { get; set; } + public string LicenseUrl { get; set; } + public bool RequiresLicenseAcceptance { get; set; } + public object Debug { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchPackageRegistration.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchPackageRegistration.cs new file mode 100644 index 000000000..3f465f335 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchPackageRegistration.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class V2SearchPackageRegistration + { + public string Id { get; set; } + public long DownloadCount { get; set; } + public bool Verified { get; set; } + public string[] Owners { get; set; } + public string[] PopularityTransfers { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchRequest.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchRequest.cs new file mode 100644 index 000000000..7d47c8cd8 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchRequest.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class V2SearchRequest : SearchRequest + { + public bool IgnoreFilter { get; set; } + public bool CountOnly { get; set; } + public V2SortBy SortBy { get; set; } + public bool LuceneQuery { get; set; } + public string PackageType { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchResponse.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchResponse.cs new file mode 100644 index 000000000..ca0eb8d82 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/V2SearchResponse.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class V2SearchResponse + { + [JsonProperty("totalHits")] + public long TotalHits { get; set; } + + [JsonProperty("data")] + public List Data { get; set; } + + public DebugInformation Debug { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/V2SortBy.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/V2SortBy.cs new file mode 100644 index 000000000..97380fbd6 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/V2SortBy.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.SearchService +{ + public enum V2SortBy + { + Popularity, + LastEditedDesc, + PublishedDesc, + SortableTitleAsc, + SortableTitleDesc, + CreatedAsc, + CreatedDesc, + TotalDownloadsAsc, + TotalDownloadsDesc, + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchContext.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchContext.cs new file mode 100644 index 000000000..10ddefd86 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchContext.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class V3SearchContext + { + [JsonProperty("@vocab")] + public string Vocab { get; set; } + + [JsonProperty("@base")] + public string Base { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchPackage.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchPackage.cs new file mode 100644 index 000000000..61faa9bd5 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchPackage.cs @@ -0,0 +1,67 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch.SearchService +{ + /// + /// Source: https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-result + /// + public class V3SearchPackage + { + [JsonProperty("@id")] + public string AtId { get; set; } + + [JsonProperty("@type")] + public string Type { get; set; } + + [JsonProperty("registration")] + public string Registration { get; set; } + + [JsonProperty("id")] + public string Id { get; set; } + + [JsonProperty("version")] + public string Version { get; set; } + + [JsonProperty("description")] + public string Description { get; set; } + + [JsonProperty("summary")] + public string Summary { get; set; } + + [JsonProperty("title")] + public string Title { get; set; } + + [JsonProperty("iconUrl")] + public string IconUrl { get; set; } + + [JsonProperty("licenseUrl")] + public string LicenseUrl { get; set; } + + [JsonProperty("projectUrl")] + public string ProjectUrl { get; set; } + + [JsonProperty("tags")] + public string[] Tags { get; set; } + + [JsonProperty("authors")] + public string[] Authors { get; set; } + + [JsonProperty("totalDownloads")] + public long TotalDownloads { get; set; } + + [JsonProperty("verified")] + public bool Verified { get; set; } + + [JsonProperty("packageTypes")] + public List PackageTypes { get; set; } + + [JsonProperty("versions")] + public List Versions { get; set; } + + public object Debug { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchPackageType.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchPackageType.cs new file mode 100644 index 000000000..ee596cc27 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchPackageType.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch.SearchService +{ + /// + /// Source: https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-result + /// + public class V3SearchPackageType + { + [JsonProperty("name")] + public string Name { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchRequest.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchRequest.cs new file mode 100644 index 000000000..7290dd258 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchRequest.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.SearchService +{ + /// + /// Source: https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#request-parameters + /// + public class V3SearchRequest : SearchRequest + { + public string PackageType { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchResponse.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchResponse.cs new file mode 100644 index 000000000..f9eb9d77c --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchResponse.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch.SearchService +{ + /// + /// Source: https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#response + /// + public class V3SearchResponse + { + [JsonProperty("@context")] + public V3SearchContext Context { get; set; } + + [JsonProperty("totalHits")] + public long TotalHits { get; set; } + + [JsonProperty("data")] + public List Data { get; set; } + + public DebugInformation Debug { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchVersion.cs b/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchVersion.cs new file mode 100644 index 000000000..8aa1ccdb5 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/Models/V3SearchVersion.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch.SearchService +{ + /// + /// Source: https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-result + /// See the section about each item in the versions array. + /// + public class V3SearchVersion + { + [JsonProperty("version")] + public string Version { get; set; } + + [JsonProperty("downloads")] + public long Downloads { get; set; } + + [JsonProperty("@id")] + public string AtId { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/NuGetQueryParser.cs b/src/NuGet.Services.AzureSearch/SearchService/NuGetQueryParser.cs new file mode 100644 index 000000000..11c533ab7 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/NuGetQueryParser.cs @@ -0,0 +1,186 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class NuGetQueryParser + { + /// + /// These words have special meaning that we will discard. + /// https://docs.microsoft.com/en-us/azure/search/query-lucene-syntax#bkmk_boolean + /// + private static readonly HashSet SpecialWords = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "AND", + "OR", + "NOT", + }; + + private static readonly IReadOnlyDictionary _queryFieldNames = new Dictionary + { + { QueryField.Id, new [] { "id" } }, + { QueryField.PackageId, new [] { "packageid" } }, + { QueryField.Version, new [] { "version" } }, + { QueryField.Title, new [] { "title" } }, + { QueryField.Description, new [] { "description" } }, + { QueryField.Tag, new [] { "tag", "tags" } }, + { QueryField.Author, new [] { "author", "authors" } }, + { QueryField.Summary, new [] { "summary" } }, + { QueryField.Owner, new [] { "owner", "owners" } }, + { QueryField.Any, new [] { "*" } } + }; + + public Dictionary> ParseQuery(string query, bool skipWhiteSpace = false) + { + var grouping = new Dictionary>(); + foreach (Clause clause in MakeClauses(Tokenize(query))) + { + if (skipWhiteSpace && string.IsNullOrWhiteSpace(clause.Text)) + { + continue; + } + + HashSet text; + var queryField = GetQueryField(clause.Field); + if (!grouping.TryGetValue(queryField, out text)) + { + text = new HashSet(); + grouping.Add(queryField, text); + } + text.Add(clause.Text); + } + + return grouping; + } + + private QueryField GetQueryField(string field) + { + foreach (var queryFieldName in _queryFieldNames) + { + if (queryFieldName.Value.Any( + s => string.Compare(s, field, StringComparison.InvariantCultureIgnoreCase) == 0)) + { + return queryFieldName.Key; + } + } + + return QueryField.Invalid; + } + + private static IEnumerable MakeClauses(IEnumerable tokens) + { + string field = null; + + foreach (Token token in tokens) + { + if (token.Type == Token.TokenType.Keyword) + { + field = token.Value; + } + else if (token.Type == Token.TokenType.Value) + { + if (SpecialWords.Contains(token.Value)) + { + continue; + } + if (field != null) + { + yield return new Clause { Field = field, Text = token.Value }; + } + else + { + yield return new Clause { Field = "*", Text = token.Value }; + } + field = null; + } + } + + yield break; + } + + private static IEnumerable Tokenize(string s) + { + var buf = new StringBuilder(); + + int state = 0; + bool previousTokenIsKeyword = false; + + foreach (char ch in s) + { + switch (state) + { + case 0: + if (Char.IsWhiteSpace(ch)) + { + if (buf.Length > 0) + { + yield return new Token { Type = Token.TokenType.Value, Value = buf.ToString() }; + previousTokenIsKeyword = false; + buf.Clear(); + } + } + else if (ch == '"') + { + state = 1; + } + else if (ch == ':') + { + if (buf.Length > 0) + { + yield return new Token { Type = Token.TokenType.Keyword, Value = buf.ToString() }; + previousTokenIsKeyword = true; + buf.Clear(); + } + } + else + { + buf.Append(ch); + } + break; + case 1: + if (ch == '"') + { + if (buf.Length > 0 || previousTokenIsKeyword) + { + yield return new Token { Type = Token.TokenType.Value, Value = buf.ToString() }; + previousTokenIsKeyword = false; + buf.Clear(); + } + state = 0; + } + else + { + buf.Append(ch); + } + break; + } + } + + if (buf.Length > 0) + { + yield return new Token { Type = Token.TokenType.Value, Value = buf.ToString() }; + previousTokenIsKeyword = false; + } + + yield break; + } + + private class Token + { + public enum TokenType { Value, Keyword } + public TokenType Type { get; set; } + public string Value { get; set; } + } + + private class Clause + { + public string Field { get; set; } + public string Text { get; set; } + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/SearchService/ParsedQuery.cs b/src/NuGet.Services.AzureSearch/SearchService/ParsedQuery.cs new file mode 100644 index 000000000..bb6dcfd04 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/ParsedQuery.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; + +namespace NuGet.Services.AzureSearch.SearchService +{ + /// + /// Contains the parsed in-memory model of the user's query. + /// + public class ParsedQuery + { + public ParsedQuery(Dictionary> grouping) + { + Grouping = grouping ?? throw new ArgumentNullException(nameof(grouping)); + } + + public Dictionary> Grouping { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/QueryField.cs b/src/NuGet.Services.AzureSearch/SearchService/QueryField.cs new file mode 100644 index 000000000..9f150d218 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/QueryField.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.SearchService +{ + public enum QueryField + { + Id, + PackageId, + Version, + Title, + Description, + Tag, + Author, + Summary, + Owner, + Any, + Invalid + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/SearchService/SearchParametersBuilder.cs b/src/NuGet.Services.AzureSearch/SearchService/SearchParametersBuilder.cs new file mode 100644 index 000000000..732afea84 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/SearchParametersBuilder.cs @@ -0,0 +1,220 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Azure.Search.Models; +using NuGet.Packaging; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class SearchParametersBuilder : ISearchParametersBuilder + { + public const int DefaultTake = 20; + private const int MaximumTake = 1000; + private const string Score = "search.score()"; + private const string Asc = " asc"; + private const string Desc = " desc"; + + private static readonly List LastCommitTimestampSelect = new List { IndexFields.LastCommitTimestamp }; + private static readonly List PackageIdsAutocompleteSelect = new List { IndexFields.PackageId }; + private static readonly List PackageVersionsAutocompleteSelect = new List { IndexFields.Search.Versions }; + + private static readonly List LastCommitTimestampDescending = new List { IndexFields.LastCommitTimestamp + Desc }; // Most recently added to the catalog first + + /// + /// We use the created timestamp as a tie-breaker since it does not change for a given package. + /// See: https://stackoverflow.com/a/34234258/52749 + /// + private static readonly List ScoreDesc = new List { Score + Desc, IndexFields.Created + Desc }; // Highest score first ("most relevant"), then newest + private static readonly List LastEditedDesc = new List { IndexFields.LastEdited + Desc, IndexFields.Created + Desc }; // Most recently edited first, then newest + private static readonly List PublishedDesc = new List { IndexFields.Published + Desc, IndexFields.Created + Desc }; // Most recently published first, then newest + private static readonly List SortableTitleAsc = new List { IndexFields.SortableTitle + Asc, IndexFields.Created + Asc }; // First title by lex order first, then oldest + private static readonly List SortableTitleDesc = new List { IndexFields.SortableTitle + Desc, IndexFields.Created + Desc }; // Last title by lex order first, then newest + private static readonly List CreatedAsc = new List { IndexFields.Created + Asc }; // Newest first + private static readonly List CreatedDesc = new List { IndexFields.Created + Desc }; // Oldest first + private static readonly List TotalDownloadsAsc = new List { IndexFields.Search.TotalDownloadCount + Asc, IndexFields.Created + Asc }; // Least downloads first, then oldest + private static readonly List TotalDownloadsDesc = new List { IndexFields.Search.TotalDownloadCount + Desc, IndexFields.Created + Desc}; // Most downloads first, then newest + + public SearchParameters LastCommitTimestamp() + { + return new SearchParameters + { + QueryType = QueryType.Full, + Select = LastCommitTimestampSelect, + OrderBy = LastCommitTimestampDescending, + Skip = 0, + Top = 1, + }; + } + + public SearchParameters V2Search(V2SearchRequest request, bool isDefaultSearch) + { + var searchParameters = NewSearchParameters(); + + if (request.CountOnly) + { + searchParameters.Skip = 0; + searchParameters.Top = 0; + searchParameters.OrderBy = null; + } + else + { + ApplyPaging(searchParameters, request); + searchParameters.OrderBy = GetOrderBy(request.SortBy); + } + + if (request.IgnoreFilter) + { + // Note that the prerelease flag has no effect when IgnoreFilter is true. + + if (!request.IncludeSemVer2) + { + searchParameters.Filter = $"{IndexFields.SemVerLevel} ne {SemVerLevelKey.SemVer2}"; + } + } + else + { + ApplySearchIndexFilter(searchParameters, request, isDefaultSearch, request.PackageType); + } + + return searchParameters; + } + + public SearchParameters V3Search(V3SearchRequest request, bool isDefaultSearch) + { + var searchParameters = NewSearchParameters(); + + ApplyPaging(searchParameters, request); + ApplySearchIndexFilter(searchParameters, request, isDefaultSearch, request.PackageType); + + return searchParameters; + } + + public SearchParameters Autocomplete(AutocompleteRequest request, bool isDefaultSearch) + { + var searchParameters = NewSearchParameters(); + + ApplySearchIndexFilter(searchParameters, request, isDefaultSearch, request.PackageType); + + switch (request.Type) + { + case AutocompleteRequestType.PackageIds: + searchParameters.Select = PackageIdsAutocompleteSelect; + ApplyPaging(searchParameters, request); + break; + + // Package version autocomplete should only match a single document + // regardless of the request's parameters. + case AutocompleteRequestType.PackageVersions: + searchParameters.Select = PackageVersionsAutocompleteSelect; + searchParameters.Skip = 0; + searchParameters.Top = 1; + break; + + default: + throw new InvalidOperationException($"Unknown autocomplete request type '{request.Type}'"); + } + + return searchParameters; + } + + private static SearchParameters NewSearchParameters() + { + return new SearchParameters + { + IncludeTotalResultCount = true, + QueryType = QueryType.Full, + OrderBy = ScoreDesc, + }; + } + + private static void ApplyPaging(SearchParameters searchParameters, SearchRequest request) + { + searchParameters.Skip = request.Skip < 0 ? 0 : request.Skip; + searchParameters.Top = request.Take < 0 || request.Take > MaximumTake ? DefaultTake : request.Take; + } + + private void ApplySearchIndexFilter( + SearchParameters searchParameters, + SearchRequest request, + bool excludePackagesHiddenByDefault, + string packageType) + { + var searchFilters = GetSearchFilters(request); + + var filterString = $"{IndexFields.Search.SearchFilters} eq '{DocumentUtilities.GetSearchFilterString(searchFilters)}'"; + + if (excludePackagesHiddenByDefault) + { + filterString += $" and ({IndexFields.Search.IsExcludedByDefault} eq false or {IndexFields.Search.IsExcludedByDefault} eq null)"; + } + + // Verify that the package type only has valid package ID characters so we don't need to worry about + // escaping quotes and such. + if (packageType != null && PackageIdValidator.IsValidPackageId(packageType)) + { + filterString += $" and {IndexFields.Search.FilterablePackageTypes}/any(p: p eq '{packageType.ToLowerInvariant()}')"; + } + + searchParameters.Filter = filterString; + } + + public SearchFilters GetSearchFilters(SearchRequest request) + { + var searchFilters = SearchFilters.Default; + + if (request.IncludePrerelease) + { + searchFilters |= SearchFilters.IncludePrerelease; + } + + if (request.IncludeSemVer2) + { + searchFilters |= SearchFilters.IncludeSemVer2; + } + + return searchFilters; + } + + private static IList GetOrderBy(V2SortBy sortBy) + { + IList orderBy; + switch (sortBy) + { + case V2SortBy.Popularity: + orderBy = ScoreDesc; + break; + case V2SortBy.LastEditedDesc: + orderBy = LastEditedDesc; + break; + case V2SortBy.PublishedDesc: + orderBy = PublishedDesc; + break; + case V2SortBy.SortableTitleAsc: + orderBy = SortableTitleAsc; + break; + case V2SortBy.SortableTitleDesc: + orderBy = SortableTitleDesc; + break; + case V2SortBy.CreatedAsc: + orderBy = CreatedAsc; + break; + case V2SortBy.CreatedDesc: + orderBy = CreatedDesc; + break; + case V2SortBy.TotalDownloadsAsc: + orderBy = TotalDownloadsAsc; + break; + case V2SortBy.TotalDownloadsDesc: + orderBy = TotalDownloadsDesc; + break; + default: + throw new ArgumentException($"The provided {nameof(V2SortBy)} is not supported.", nameof(sortBy)); + } + + return orderBy; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/SearchResponseBuilder.cs b/src/NuGet.Services.AzureSearch/SearchService/SearchResponseBuilder.cs new file mode 100644 index 000000000..9fa620a21 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/SearchResponseBuilder.cs @@ -0,0 +1,553 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Options; +using NuGet.Protocol.Registration; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class SearchResponseBuilder : ISearchResponseBuilder + { + private static readonly V2SearchDependency[] EmptyDependencies = new V2SearchDependency[0]; + private readonly Lazy _lazyAuxiliaryData; + private readonly IOptionsSnapshot _options; + private readonly FlatContainerPackagePathProvider _pathProvider; + private readonly Uri _flatContainerBaseUrl; + + public SearchResponseBuilder( + Lazy auxiliaryData, + IOptionsSnapshot options) + { + _lazyAuxiliaryData = auxiliaryData ?? throw new ArgumentNullException(nameof(auxiliaryData)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + + if (_options.Value.SemVer1RegistrationsBaseUrl == null) + { + throw new ArgumentException($"The {nameof(SearchServiceConfiguration.SemVer1RegistrationsBaseUrl)} needs to be set.", nameof(options)); + } + + if (_options.Value.SemVer2RegistrationsBaseUrl == null) + { + throw new ArgumentException($"The {nameof(SearchServiceConfiguration.SemVer2RegistrationsBaseUrl)} needs to be set.", nameof(options)); + } + + if (_options.Value.AllIconsInFlatContainer) + { + if (_options.Value.FlatContainerBaseUrl == null) + { + throw new ArgumentException($"The {nameof(SearchServiceConfiguration.FlatContainerBaseUrl)} needs to be set.", nameof(options)); + } + + if (_options.Value.FlatContainerContainerName == null) + { + throw new ArgumentException($"The {nameof(SearchServiceConfiguration.FlatContainerContainerName)} needs to be set.", nameof(options)); + } + + _pathProvider = new FlatContainerPackagePathProvider(_options.Value.FlatContainerContainerName); + _flatContainerBaseUrl = _options.Value.ParseFlatContainerBaseUrl(); + } + } + + private IAuxiliaryData AuxiliaryData => _lazyAuxiliaryData.Value; + + public V2SearchResponse V2FromHijack( + V2SearchRequest request, + string text, + SearchParameters searchParameters, + DocumentSearchResult result, + TimeSpan duration) + { + return ToResponse( + request, + searchParameters, + text, + _options.Value.HijackIndexName, + result, + duration, + p => ToV2SearchPackage(p, request.IncludeSemVer2)); + } + + public V2SearchResponse V2FromSearch( + V2SearchRequest request, + string text, + SearchParameters parameters, + DocumentSearchResult result, + TimeSpan duration) + { + return ToResponse( + request, + parameters, + text, + _options.Value.SearchIndexName, + result, + duration, + p => ToV2SearchPackage(p)); + } + + public V2SearchResponse V2FromSearchDocument( + V2SearchRequest request, + string documentKey, + SearchDocument.Full document, + TimeSpan duration) + { + return ToResponse( + request, + _options.Value.SearchIndexName, + documentKey, + document, + duration, + p => ToV2SearchPackage(p)); + } + + public V2SearchResponse V2FromHijackDocument( + V2SearchRequest request, + string documentKey, + HijackDocument.Full document, + TimeSpan duration) + { + return ToResponse( + request, + _options.Value.HijackIndexName, + documentKey, + document, + duration, + p => ToV2SearchPackage(p, request.IncludeSemVer2)); + } + + public V3SearchResponse V3FromSearchDocument( + V3SearchRequest request, + string documentKey, + SearchDocument.Full document, + TimeSpan duration) + { + var registrationsBaseUrl = GetRegistrationsBaseUrl(request.IncludeSemVer2); + + var data = new List(); + if (document != null) + { + var package = ToV3SearchPackage(document, registrationsBaseUrl); + package.Debug = request.ShowDebug ? new DebugDocumentResult { Document = document } : null; + data.Add(package); + } + + return new V3SearchResponse + { + Context = new V3SearchContext + { + Vocab = "http://schema.nuget.org/schema#", + Base = registrationsBaseUrl, + }, + TotalHits = data.Count, + Data = data, + Debug = DebugInformation.CreateFromGetOrNull( + request, + _options.Value.SearchIndexName, + documentKey, + duration, + AuxiliaryData.Metadata), + }; + } + + public V3SearchResponse V3FromSearch( + V3SearchRequest request, + string text, + SearchParameters parameters, + DocumentSearchResult result, + TimeSpan duration) + { + var results = result.Results; + result.Results = null; + + var registrationsBaseUrl = GetRegistrationsBaseUrl(request.IncludeSemVer2); + + return new V3SearchResponse + { + Context = GetV3SearchContext(registrationsBaseUrl), + TotalHits = result.Count.Value, + Data = results + .Select(x => + { + var package = ToV3SearchPackage(x.Document, registrationsBaseUrl); + package.Debug = request.ShowDebug ? x : null; + return package; + }) + .ToList(), + Debug = DebugInformation.CreateFromSearchOrNull( + request, + _options.Value.SearchIndexName, + parameters, + text, + result, + duration, + AuxiliaryData.Metadata), + }; + } + + public AutocompleteResponse AutocompleteFromSearch( + AutocompleteRequest request, + string text, + SearchParameters parameters, + DocumentSearchResult result, + TimeSpan duration) + { + var results = result.Results; + result.Results = null; + + List data; + switch (request.Type) + { + case AutocompleteRequestType.PackageIds: + data = results.Select(x => x.Document.PackageId).ToList(); + break; + + case AutocompleteRequestType.PackageVersions: + if (result.Count > 1 || results.Count > 1) + { + throw new ArgumentException( + "Package version autocomplete queries should have a single document result", + nameof(result)); + } + + data = results.SelectMany(x => x.Document.Versions).ToList(); + break; + + default: + throw new InvalidOperationException($"Unknown autocomplete request type '{request.Type}'"); + } + + return new AutocompleteResponse + { + Context = GetAutocompleteContext(), + TotalHits = result.Count.Value, + Data = data, + Debug = DebugInformation.CreateFromSearchOrNull( + request, + _options.Value.SearchIndexName, + parameters, + text, + result, + duration, + auxiliaryFilesMetadata: null), + }; + } + + private static string TitleThenId(IBaseMetadataDocument document) + { + if (!string.IsNullOrWhiteSpace(document.Title)) + { + return document.Title; + } + + return document.PackageId; + } + + private V2SearchResponse ToResponse( + V2SearchRequest request, + string indexName, + string documentKey, + T document, + TimeSpan duration, + Func toPackage) + where T : class + { + var data = new List(); + if (document != null) + { + var package = toPackage(document); + package.Debug = request.ShowDebug ? new DebugDocumentResult { Document = document } : null; + data.Add(package); + } + + if (request.CountOnly) + { + return new V2SearchResponse + { + TotalHits = data.Count, + Debug = DebugInformation.CreateFromGetOrNull( + request, + indexName, + documentKey, + duration, + AuxiliaryData.Metadata), + }; + } + else + { + return new V2SearchResponse + { + TotalHits = data.Count, + Data = data, + Debug = DebugInformation.CreateFromGetOrNull( + request, + indexName, + documentKey, + duration, + AuxiliaryData.Metadata), + }; + } + } + + private V2SearchResponse ToResponse( + V2SearchRequest request, + SearchParameters parameters, + string text, + string indexName, + DocumentSearchResult result, + TimeSpan duration, + Func toPackage) + where T : class + { + var results = result.Results; + result.Results = null; + + if (request.CountOnly) + { + return new V2SearchResponse + { + TotalHits = result.Count.Value, + Debug = DebugInformation.CreateFromSearchOrNull( + request, + indexName, + parameters, + text, + result, + duration, + AuxiliaryData.Metadata), + }; + } + else + { + var resultData = results.Select(x => + { + var package = toPackage(x.Document); + package.Debug = request.ShowDebug ? x : null; + return package; + }); + + // The real sorting happens in Azure Search. However, we do another round of sorting here since the sorting + // on Azure Search index's downloads may produce different results from what customers see on + // the Gallery (which uses Auxiliary file's download count) + if (request.SortBy == V2SortBy.TotalDownloadsAsc) + { + resultData = resultData.OrderBy(x => x.PackageRegistration.DownloadCount).ThenBy(x => x.Created); + } + else if (request.SortBy == V2SortBy.TotalDownloadsDesc) + { + resultData = resultData.OrderByDescending(x => x.PackageRegistration.DownloadCount).ThenByDescending(x => x.Created); + } + + return new V2SearchResponse + { + TotalHits = result.Count.Value, + Data = resultData + .ToList(), + Debug = DebugInformation.CreateFromSearchOrNull( + request, + indexName, + parameters, + text, + result, + duration, + AuxiliaryData.Metadata), + }; + } + } + + private V3SearchPackage ToV3SearchPackage(SearchDocument.Full result, string registrationsBaseUrl) + { + var registrationIndexUrl = RegistrationUrlBuilder.GetIndexUrl(registrationsBaseUrl, result.PackageId); + return new V3SearchPackage + { + AtId = registrationIndexUrl, + Type = "Package", + Registration = registrationIndexUrl, + Id = result.PackageId, + Version = result.FullVersion, + Description = result.Description ?? string.Empty, + Summary = result.Summary ?? string.Empty, + Title = TitleThenId(result), + IconUrl = GetIconUrl(result), + LicenseUrl = result.LicenseUrl, + ProjectUrl = result.ProjectUrl, + Tags = result.Tags ?? Array.Empty(), + Authors = new[] { result.Authors ?? string.Empty }, + TotalDownloads = AuxiliaryData.GetTotalDownloadCount(result.PackageId), + Verified = AuxiliaryData.IsVerified(result.PackageId), + PackageTypes = GetV3SearchPackageTypes(result), + Versions = result + .Versions + .Select(x => + { + // Each of these versions is the full version. + var lowerVersion = NuGetVersion.Parse(x).ToNormalizedString().ToLowerInvariant(); + return new V3SearchVersion + { + Version = x, + Downloads = AuxiliaryData.GetDownloadCount(result.PackageId, lowerVersion), + AtId = RegistrationUrlBuilder.GetLeafUrl(registrationsBaseUrl, result.PackageId, x), + }; + }) + .ToList(), + }; + } + + private List GetV3SearchPackageTypes(SearchDocument.UpdateLatest document) + { + if (document.PackageTypes == null || document.PackageTypes.Length == 0) + { + return null; + } + + var packageTypes = new List(document.PackageTypes.Length); + foreach (var packageType in document.PackageTypes) + { + packageTypes.Add(new V3SearchPackageType { Name = packageType }); + } + + return packageTypes; + } + + private string GetIconUrl(IBaseMetadataDocument document) + { + // If we mandate that all icon URLs come from flat container, generate the URL. + if (_options.Value.AllIconsInFlatContainer + && !string.IsNullOrWhiteSpace(document.IconUrl)) + { + var iconPath = _pathProvider.GetIconPath(document.PackageId, document.NormalizedVersion, normalize: false); + return new Uri(_flatContainerBaseUrl, iconPath).AbsoluteUri; + } + + return document.IconUrl; + } + + private V2SearchPackage ToV2SearchPackage(SearchDocument.Full result) + { + var package = BaseMetadataDocumentToPackage(result); + + package.PackageRegistration.Owners = result.Owners ?? Array.Empty(); + package.Listed = true; + package.IsLatestStable = result.IsLatestStable.Value; + package.IsLatest = result.IsLatest.Value; + + return package; + } + + private V2SearchPackage ToV2SearchPackage(HijackDocument.Full result, bool semVer2) + { + var package = BaseMetadataDocumentToPackage(result); + + // The owners are not used in the hijack scenarios. + package.PackageRegistration.Owners = Array.Empty(); + + package.Listed = result.Listed.Value; + package.IsLatestStable = semVer2 ? result.IsLatestStableSemVer2.Value : result.IsLatestStableSemVer1.Value; + package.IsLatest = semVer2 ? result.IsLatestSemVer2.Value : result.IsLatestSemVer1.Value; + + return package; + } + + private V2SearchPackage BaseMetadataDocumentToPackage(IBaseMetadataDocument document) + { + return new V2SearchPackage + { + PackageRegistration = new V2SearchPackageRegistration + { + Id = document.PackageId, + DownloadCount = AuxiliaryData.GetTotalDownloadCount(document.PackageId), + Verified = AuxiliaryData.IsVerified(document.PackageId), + PopularityTransfers = AuxiliaryData.GetPopularityTransfers(document.PackageId), + }, + Version = document.OriginalVersion ?? document.NormalizedVersion, + NormalizedVersion = document.NormalizedVersion, + Title = TitleThenId(document), + Description = document.Description ?? string.Empty, + Summary = document.Summary ?? string.Empty, + Authors = document.Authors ?? string.Empty, + Copyright = document.Copyright, + Language = document.Language, + Tags = document.Tags != null ? string.Join(" ", document.Tags) : string.Empty, + ReleaseNotes = document.ReleaseNotes, + ProjectUrl = document.ProjectUrl, + IconUrl = GetIconUrl(document), + Created = document.Created.Value, + Published = document.Published.Value, + LastUpdated = document.Published.Value, + LastEdited = document.LastEdited, + DownloadCount = AuxiliaryData.GetDownloadCount(document.PackageId, document.NormalizedVersion), + FlattenedDependencies = document.FlattenedDependencies, + Dependencies = EmptyDependencies, + SupportedFrameworks = Array.Empty(), + MinClientVersion = document.MinClientVersion, + Hash = document.Hash, + HashAlgorithm = document.HashAlgorithm, + PackageFileSize = document.FileSize.Value, + LicenseUrl = document.LicenseUrl, + RequiresLicenseAcceptance = document.RequiresLicenseAcceptance ?? false, + }; + } + + public string GetRegistrationsBaseUrl(bool includeSemVer2) + { + var url = includeSemVer2 ? _options.Value.SemVer2RegistrationsBaseUrl : _options.Value.SemVer1RegistrationsBaseUrl; + + return url.TrimEnd('/') + '/'; + } + + public V2SearchResponse EmptyV2(V2SearchRequest request) + { + return new V2SearchResponse + { + TotalHits = 0, + Data = new List(), + Debug = DebugInformation.CreateFromEmptyOrNull(request), + }; + } + + public V3SearchResponse EmptyV3(V3SearchRequest request) + { + var registrationsBaseUrl = GetRegistrationsBaseUrl(request.IncludeSemVer2); + + return new V3SearchResponse + { + Context = GetV3SearchContext(registrationsBaseUrl), + TotalHits = 0, + Data = new List(), + Debug = DebugInformation.CreateFromEmptyOrNull(request), + }; + } + + public AutocompleteResponse EmptyAutocomplete(AutocompleteRequest request) + { + return new AutocompleteResponse + { + Context = GetAutocompleteContext(), + TotalHits = 0, + Data = new List(), + Debug = DebugInformation.CreateFromEmptyOrNull(request), + }; + } + + private static V3SearchContext GetV3SearchContext(string registrationsBaseUrl) + { + return new V3SearchContext + { + Vocab = "http://schema.nuget.org/schema#", + Base = registrationsBaseUrl, + }; + } + + private static AutocompleteContext GetAutocompleteContext() + { + return new AutocompleteContext + { + Vocab = "http://schema.nuget.org/schema#", + }; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/SearchServiceConfiguration.cs b/src/NuGet.Services.AzureSearch/SearchService/SearchServiceConfiguration.cs new file mode 100644 index 000000000..d206e05f9 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/SearchServiceConfiguration.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class SearchServiceConfiguration : AzureSearchConfiguration + { + public float MatchAllTermsBoost { get; set; } = 3; + public float PrefixMatchBoost { get; set; } = 20; + public float ExactMatchBoost { get; set; } = 1000; + public string SemVer1RegistrationsBaseUrl { get; set; } + public string SemVer2RegistrationsBaseUrl { get; set; } + public TimeSpan AuxiliaryDataReloadFrequency { get; set; } = TimeSpan.FromHours(1); + public TimeSpan AuxiliaryDataReloadFailureRetryFrequency { get; set; } = TimeSpan.FromSeconds(30); + public TimeSpan SecretRefreshFrequency { get; set; } = TimeSpan.FromHours(12); + public TimeSpan SecretRefreshFailureRetryFrequency { get; set; } = TimeSpan.FromMinutes(5); + public string DeploymentLabel { get; set; } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/SearchStatusOptions.cs b/src/NuGet.Services.AzureSearch/SearchService/SearchStatusOptions.cs new file mode 100644 index 000000000..86f097ce6 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/SearchStatusOptions.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch.SearchService +{ + [Flags] + public enum SearchStatusOptions + { + None = 0, + Server = 1 << 0, + AuxiliaryFiles = 1 << 1, + AzureSearch = 1 << 2, + All = ~None, + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/SearchStatusService.cs b/src/NuGet.Services.AzureSearch/SearchService/SearchStatusService.cs new file mode 100644 index 000000000..d0cd06778 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/SearchStatusService.cs @@ -0,0 +1,203 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Services.AzureSearch.Wrappers; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class SearchStatusService : ISearchStatusService + { + private readonly ISearchIndexClientWrapper _searchIndex; + private readonly ISearchIndexClientWrapper _hijackIndex; + private readonly ISearchParametersBuilder _parametersBuilder; + private readonly IAuxiliaryDataCache _auxiliaryDataCache; + private readonly ISecretRefresher _secretRefresher; + private readonly IOptionsSnapshot _options; + private readonly IAzureSearchTelemetryService _telemetryService; + private readonly ILogger _logger; + + public SearchStatusService( + ISearchIndexClientWrapper searchIndex, + ISearchIndexClientWrapper hijackIndex, + ISearchParametersBuilder parametersBuilder, + IAuxiliaryDataCache auxiliaryDataCache, + ISecretRefresher secretRefresher, + IOptionsSnapshot options, + IAzureSearchTelemetryService telemetryService, + ILogger logger) + { + _searchIndex = searchIndex ?? throw new ArgumentNullException(nameof(searchIndex)); + _hijackIndex = hijackIndex ?? throw new ArgumentNullException(nameof(hijackIndex)); + _parametersBuilder = parametersBuilder ?? throw new ArgumentNullException(nameof(parametersBuilder)); + _auxiliaryDataCache = auxiliaryDataCache ?? throw new ArgumentNullException(nameof(auxiliaryDataCache)); + _secretRefresher = secretRefresher ?? throw new ArgumentNullException(nameof(secretRefresher)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task GetStatusAsync(SearchStatusOptions options, Assembly assemblyForMetadata) + { + var response = new SearchStatusResponse + { + Success = true, + }; + + response.Duration = await Measure.DurationAsync(() => PopulateResponseAsync(options, assemblyForMetadata, response)); + + _telemetryService.TrackGetSearchServiceStatus(options, response.Success, response.Duration.Value); + + return response; + } + + private async Task PopulateResponseAsync(SearchStatusOptions options, Assembly assembly, SearchStatusResponse response) + { + await Task.WhenAll( + TryAsync( + async () => + { + if (options.HasFlag(SearchStatusOptions.AzureSearch)) + { + response.SearchIndex = await GetIndexStatusAsync(_searchIndex); + } + }, + response, + "warming the search index"), + TryAsync( + async () => + { + if (options.HasFlag(SearchStatusOptions.AzureSearch)) + { + response.HijackIndex = await GetIndexStatusAsync(_hijackIndex); + } + }, + response, + "warming the hijack index"), + TryAsync( + async () => + { + if (options.HasFlag(SearchStatusOptions.AuxiliaryFiles)) + { + response.AuxiliaryFiles = await GetAuxiliaryFilesMetadataAsync(); + } + }, + response, + "getting cached auxiliary data"), + TryAsync( + async () => + { + if (options.HasFlag(SearchStatusOptions.Server)) + { + response.Server = await GetServerStatusAsync(assembly); + } + }, + response, + "getting server information")); + } + + private async Task TryAsync( + Func getAsync, + SearchStatusResponse response, + string operation) + { + try + { + await Task.Yield(); + await getAsync(); + } + catch (Exception ex) + { + response.Success = false; + _logger.LogError(0, ex, "When getting the search status, {Operation} failed.", operation); + } + } + + private Task GetServerStatusAsync(Assembly assembly) + { + DateTimeOffset processStartTime; + int processId; + using (var process = Process.GetCurrentProcess()) + { + processStartTime = process.StartTime.ToUniversalTime(); + processId = process.Id; + } + + var lastSecretRefresh = _secretRefresher.LastRefresh; + + var serverStatus = new ServerStatus + { + AssemblyBuildDateUtc = GetAssemblyMetadataOrNull(assembly, "BuildDateUtc"), + AssemblyCommitId = GetAssemblyMetadataOrNull(assembly, "CommitId"), + AssemblyInformationalVersion = GetAssemblyInformationalVersionOrNull(assembly), + DeploymentLabel = _options.Value.DeploymentLabel, + MachineName = Environment.MachineName, + InstanceId = Environment.GetEnvironmentVariable("WEBSITE_INSTANCE_ID"), + ProcessDuration = DateTimeOffset.UtcNow - processStartTime, + ProcessId = processId, + ProcessStartTime = processStartTime, + LastServiceRefreshTime = lastSecretRefresh, + }; + + return Task.FromResult(serverStatus); + } + + private async Task GetAuxiliaryFilesMetadataAsync() + { + await _auxiliaryDataCache.EnsureInitializedAsync(); + return _auxiliaryDataCache.Get().Metadata; + } + + private static string GetAssemblyInformationalVersionOrNull(Assembly assembly) + { + return assembly + .GetCustomAttributes() + .Select(x => x.InformationalVersion) + .FirstOrDefault(); + } + + private static string GetAssemblyMetadataOrNull(Assembly assembly, string name) + { + return assembly + .GetCustomAttributes() + .Where(x => x.Key == name) + .Select(x => x.Value) + .FirstOrDefault(); + } + + private async Task GetIndexStatusAsync(ISearchIndexClientWrapper index) + { + var documentCountResult = await Measure.DurationWithValueAsync(() => index.Documents.CountAsync()); + _telemetryService.TrackDocumentCountQuery(index.IndexName, documentCountResult.Value, documentCountResult.Duration); + + var lastCommitTimestampParameters = _parametersBuilder.LastCommitTimestamp(); + var lastCommitTimestampResult = await Measure.DurationWithValueAsync(() => index.Documents.SearchAsync("*", lastCommitTimestampParameters)); + var lastCommitTimestamp = lastCommitTimestampResult + .Value? + .Results? + .FirstOrDefault()? + .Document[IndexFields.LastCommitTimestamp] as DateTimeOffset?; + _telemetryService.TrackLastCommitTimestampQuery(index.IndexName, lastCommitTimestamp, lastCommitTimestampResult.Duration); + + var warmQueryDuration = await Measure.DurationAsync(() => index.Documents.SearchAsync("*", new SearchParameters())); + _telemetryService.TrackWarmQuery(index.IndexName, warmQueryDuration); + + return new IndexStatus + { + DocumentCount = documentCountResult.Value, + DocumentCountDuration = documentCountResult.Duration, + Name = index.IndexName, + WarmQueryDuration = warmQueryDuration, + LastCommitTimestamp = lastCommitTimestamp, + LastCommitTimestampDuration = lastCommitTimestampResult.Duration, + }; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/SearchTextBuilder.cs b/src/NuGet.Services.AzureSearch/SearchService/SearchTextBuilder.cs new file mode 100644 index 000000000..1ad324302 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/SearchTextBuilder.cs @@ -0,0 +1,310 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Options; +using NuGet.Packaging; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public partial class SearchTextBuilder : ISearchTextBuilder + { + public const string MatchAllDocumentsQuery = "*"; + private static readonly char[] PackageIdSeparators = new[] { '.', '-', '_' }; + private static readonly char[] TokenizationSeparators = new[] { '.', '-', '_', ',' }; + private static readonly Regex TokenizePackageIdRegex = new Regex( + @"((?<=[a-z])(?=[A-Z])|((?<=[0-9])(?=[A-Za-z]))|((?<=[A-Za-z])(?=[0-9]))|[.\-_,])", + RegexOptions.None, + matchTimeout: TimeSpan.FromSeconds(10)); + + private static readonly IReadOnlyDictionary FieldNames = new Dictionary + { + { QueryField.Author, IndexFields.Authors }, + { QueryField.Description, IndexFields.Description }, + { QueryField.Id, IndexFields.TokenizedPackageId }, + { QueryField.Owner, IndexFields.Search.Owners }, + { QueryField.PackageId, IndexFields.PackageId }, + { QueryField.Summary, IndexFields.Summary }, + { QueryField.Tag, IndexFields.Tags }, + { QueryField.Title, IndexFields.Title }, + { QueryField.Version, IndexFields.NormalizedVersion }, + }; + + private readonly IOptionsSnapshot _options; + private readonly NuGetQueryParser _parser; + + public SearchTextBuilder(IOptionsSnapshot options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _parser = new NuGetQueryParser(); + } + + public ParsedQuery ParseV2Search(V2SearchRequest request) + { + var query = request.Query; + + // The old V2 search service would treat "id:" queries (~match) in the same way as it did "packageid:" (==match). + // If "id:" is in the query, replace it. + if (request.LuceneQuery && !string.IsNullOrEmpty(query) && query.StartsWith("id:", StringComparison.OrdinalIgnoreCase)) + { + query = "packageid:" + query.Substring(3); + } + + return GetParsedQuery(query); + } + + public ParsedQuery ParseV3Search(V3SearchRequest request) + { + return GetParsedQuery(request.Query); + } + + public string Autocomplete(AutocompleteRequest request) + { + if (string.IsNullOrWhiteSpace(request.Query)) + { + return MatchAllDocumentsQuery; + } + + // Query package ids. If autocompleting package ids, allow prefix matches. + var builder = new AzureSearchTextBuilder(); + + if (request.Type == AutocompleteRequestType.PackageIds) + { + var trimmedQuery = request.Query.Trim(); + + builder.AppendScopedTerm( + fieldName: IndexFields.PackageId, + term: trimmedQuery, + prefixSearch: true); + + var pieces = trimmedQuery.Split(PackageIdSeparators); + foreach (var piece in pieces) + { + if (string.IsNullOrWhiteSpace(piece)) + { + continue; + } + + builder.AppendScopedTerm( + fieldName: IndexFields.TokenizedPackageId, + term: piece, + required: true, + prefixSearch: true); + } + + if (IsId(trimmedQuery)) + { + builder.AppendExactMatchPackageIdBoost(trimmedQuery, _options.Value.ExactMatchBoost); + } + } + else + { + builder.AppendScopedTerm( + fieldName: IndexFields.PackageId, + term: request.Query, + prefixSearch: false); + } + + + return builder.ToString(); + } + + private ParsedQuery GetParsedQuery(string query) + { + if (string.IsNullOrWhiteSpace(query)) + { + return new ParsedQuery(new Dictionary>()); + } + + var grouping = _parser.ParseQuery(query.Trim(), skipWhiteSpace: true); + + return new ParsedQuery(grouping); + } + + public string Build(ParsedQuery parsed) + { + if (!parsed.Grouping.Any()) + { + return MatchAllDocumentsQuery; + } + + var scopedTerms = parsed.Grouping.Where(g => g.Key != QueryField.Any && g.Key != QueryField.Invalid).ToList(); + var unscopedTerms = parsed.Grouping.Where(g => g.Key == QueryField.Any) + .Select(g => g.Value) + .SingleOrDefault()? + .ToList(); + + // Don't bother generating Azure Search text if all terms are scoped to invalid fields. + var hasUnscopedTerms = unscopedTerms != null && unscopedTerms.Count > 0; + if (scopedTerms.Count == 0 && !hasUnscopedTerms) + { + return MatchAllDocumentsQuery; + } + + // Add the terms that are scoped to specific fields. + var builder = new AzureSearchTextBuilder(); + var requireScopedTerms = hasUnscopedTerms || scopedTerms.Count > 1; + + foreach (var scopedTerm in scopedTerms) + { + var fieldName = FieldNames[scopedTerm.Key]; + var values = ProcessFieldValues(scopedTerm.Key, scopedTerm.Value).ToList(); + + if (values.Count == 0) + { + // This happens if tags have only delimiters. + continue; + } + else if (values.Count > 1) + { + builder.AppendScopedTerms(fieldName, values, required: requireScopedTerms); + } + else + { + builder.AppendScopedTerm(fieldName, values.First(), required: requireScopedTerms); + } + } + + // Add the terms that can match any fields. + if (hasUnscopedTerms) + { + builder.AppendTerms(unscopedTerms); + + // Favor results that match all unscoped terms. + // We don't need to include scoped terms as these are required. + if (unscopedTerms.Count > 1) + { + builder.AppendBoostIfMatchAllTerms(unscopedTerms, _options.Value.MatchAllTermsBoost); + } + + // Try to favor results that match all unscoped terms after tokenization. + // Don't generate this clause if it is equal to or a subset of the "match all unscoped terms" clause. + var tokenizedUnscopedTerms = new HashSet(unscopedTerms.SelectMany(Tokenize)); + if (tokenizedUnscopedTerms.Count > unscopedTerms.Count || !tokenizedUnscopedTerms.All(unscopedTerms.Contains)) + { + builder.AppendBoostIfMatchAllTerms(tokenizedUnscopedTerms.ToList(), _options.Value.MatchAllTermsBoost); + } + + // Favor results that prefix match the last unscoped term for an "instant search" experience. + if (scopedTerms.Count == 0) + { + var lastUnscopedTerm = unscopedTerms.Last(); + if (IsIdWithSeparator(lastUnscopedTerm)) + { + builder.AppendScopedTerm( + fieldName: IndexFields.PackageId, + term: lastUnscopedTerm, + required: false, + prefixSearch: true, + boost: _options.Value.PrefixMatchBoost); + } + else + { + var boost = lastUnscopedTerm.Length < 4 + ? _options.Value.PrefixMatchBoost + : 1; + + builder.AppendScopedTerm( + fieldName: IndexFields.TokenizedPackageId, + term: lastUnscopedTerm, + required: false, + prefixSearch: true, + boost: boost); + } + } + } + + // Handle the exact match case. If the search query is a single unscoped term is also a valid package + // ID, mega boost the document that has this package ID. Only consider the query to be a package ID has + // symbols (a.k.a. separators) in it. + if (scopedTerms.Count == 0 + && unscopedTerms.Count == 1 + && IsIdWithSeparator(unscopedTerms[0])) + { + builder.AppendExactMatchPackageIdBoost(unscopedTerms[0], _options.Value.ExactMatchBoost); + } + + var result = builder.ToString(); + if (string.IsNullOrWhiteSpace(result)) + { + return MatchAllDocumentsQuery; + } + + return result; + } + + private static IEnumerable ProcessFieldValues(QueryField field, IEnumerable values) + { + switch (field) + { + // Expand tags by their delimiters + case QueryField.Tag: + return values.SelectMany(Utils.SplitTags).Distinct(); + + // The "version" query field should be normalized if possible. + case QueryField.Version: + return values.Select(value => + { + if (!NuGetVersion.TryParse(value, out var version)) + { + return value; + } + + return version.ToNormalizedString(); + }); + + default: + return values; + } + } + + private static bool IsId(string query) + { + return query.Length <= PackageIdValidator.MaxPackageIdLength + && PackageIdValidator.IsValidPackageId(query); + } + + private static bool IsIdWithSeparator(string query) + { + return query.IndexOfAny(PackageIdSeparators) >= 0 && IsId(query); + } + + /// + /// Tokenizes terms. This is similar to with the following differences: + /// + /// 1. Does not split terms on whitespace + /// 2. Does not split terms on the following characters: ' ; : * # ! ~ + ( ) [ ] { } + /// + /// The input to tokenize + /// The tokens extracted from the inputted term + private static IReadOnlyList Tokenize(string term) + { + // Don't tokenize phrases. These are multiple terms that were wrapped in quotes. + if (term.Any(char.IsWhiteSpace)) + { + return new List { term }; + } + + return TokenizePackageIdRegex + .Split(term) + .Where(t => !string.IsNullOrEmpty(t)) + .Where(t => !IsTokenizationSeparator(t)) + .ToList(); + } + + private static bool IsTokenizationSeparator(string input) + { + if (input.Length != 1) + { + return false; + } + + return TokenizationSeparators.Any(separator => input[0] == separator); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/SearchService/SecretRefresher.cs b/src/NuGet.Services.AzureSearch/SearchService/SecretRefresher.cs new file mode 100644 index 000000000..5c57a61f9 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/SearchService/SecretRefresher.cs @@ -0,0 +1,71 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Services.AzureSearch.Wrappers; +using NuGet.Services.KeyVault; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class SecretRefresher : ISecretRefresher + { + private readonly IRefreshableSecretReaderFactory _factory; + private readonly ISystemTime _systemTime; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public SecretRefresher( + IRefreshableSecretReaderFactory factory, + ISystemTime systemTime, + IOptionsSnapshot options, + ILogger logger) + { + _factory = factory ?? throw new ArgumentNullException(nameof(factory)); + _systemTime = systemTime ?? throw new ArgumentNullException(nameof(systemTime)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + /// We can initialize the "last refresh" time to the current time since secrets are loaded as the app is + /// starting. + /// + public DateTimeOffset LastRefresh { get; private set; } = DateTimeOffset.UtcNow; + + public async Task RefreshContinuouslyAsync(CancellationToken token) + { + while (!token.IsCancellationRequested) + { + _logger.LogInformation("Trying to refresh the secrets."); + + TimeSpan delay; + try + { + await _factory.RefreshAsync(token); + delay = _options.Value.SecretRefreshFrequency; + LastRefresh = DateTimeOffset.UtcNow; + } + catch (Exception ex) + { + _logger.LogError(0, ex, "An exception was thrown while refreshing the secrets."); + + delay = _options.Value.SecretRefreshFailureRetryFrequency; + } + + if (token.IsCancellationRequested) + { + return; + } + + _logger.LogInformation( + "Waiting {Duration} before attempting to refresh the secrets again.", + delay); + await _systemTime.Delay(delay, token); + } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/ServiceClientTracingLogger.cs b/src/NuGet.Services.AzureSearch/ServiceClientTracingLogger.cs new file mode 100644 index 000000000..ff548cb17 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/ServiceClientTracingLogger.cs @@ -0,0 +1,60 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Rest; + +namespace NuGet.Services.AzureSearch +{ + public class ServiceClientTracingLogger : IServiceClientTracingInterceptor + { + private const string Prefix = "ServiceClient "; + private readonly ILogger _logger; + + public ServiceClientTracingLogger(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public void SendRequest(string invocationId, HttpRequestMessage request) + { + _logger.LogInformation( + Prefix + "invocation {InvocationId} sending request: {Method} {RequestUri}", + invocationId, + request.Method, + request.RequestUri); + } + + public void ReceiveResponse(string invocationId, HttpResponseMessage response) + { + _logger.LogInformation( + Prefix + "invocation {InvocationId} received response: {StatusCode} {ReasonPhrase}", + invocationId, + (int)response.StatusCode, + response.ReasonPhrase); + } + + public void Configuration(string source, string name, string value) + { + } + + public void EnterMethod(string invocationId, object instance, string method, IDictionary parameters) + { + } + + public void ExitMethod(string invocationId, object returnValue) + { + } + + public void Information(string message) + { + } + + public void TraceError(string invocationId, Exception exception) + { + } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/FilteredVersionList.cs b/src/NuGet.Services.AzureSearch/VersionList/FilteredVersionList.cs new file mode 100644 index 000000000..bda770620 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/FilteredVersionList.cs @@ -0,0 +1,267 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch +{ + /// + /// This type tracks a version list for a specific . In other words, this implementation + /// assumes that a non-applicable version is never added to the list. In practice, does + /// this filtering before adding a version to this list. + /// + internal class FilteredVersionList + { + internal readonly SortedList _versions; + internal NuGetVersion _latestOrNull; + + public FilteredVersionList(IEnumerable versions) + { + if (versions == null) + { + throw new ArgumentNullException(nameof(versions)); + } + + _versions = new SortedList(); + + foreach (var version in versions) + { + _versions[version.ParsedVersion] = version; + } + + _latestOrNull = CalculateLatest(); + } + + public LatestVersionInfo GetLatestVersionInfo() + { + if (_latestOrNull == null) + { + return null; + } + + return new LatestVersionInfo( + _latestOrNull, + _versions[_latestOrNull].FullVersion, + _versions + .Where(x => x.Value.Listed) + .Select(x => x.Value.FullVersion) + .ToArray()); + } + + public LatestIndexChanges Delete(NuGetVersion deleted) + { + var ctx = UpdateVersionList( + (v, p) => _versions.Remove(v), + deleted, + newProperties: null); + + var searchIndexChangeType = DeleteFromSearchIndex(ctx); + var hijackIndexChanges = DeleteFromHijackIndex(ctx); + + return new LatestIndexChanges(searchIndexChangeType, hijackIndexChanges); + } + + /// + /// When a non-applicable version is encountered, the search index should make sure it doesn't have that + /// version at all (much like a ). For the hijack index, the non-applicable + /// version should not be deleted but the latest booleans should still be updated, for reflow. + /// + public LatestIndexChanges Remove(NuGetVersion version) + { + var ctx = UpdateVersionList( + (v, p) => _versions.Remove(v), + version, + newProperties: null); + + var searchIndexChangeType = UpsertToSearchIndex(ctx); + var hijackIndexChanges = UpsertToHijackIndex(ctx); + + return new LatestIndexChanges(searchIndexChangeType, hijackIndexChanges); + } + + public LatestIndexChanges Upsert(FilteredVersionProperties addedProperties) + { + var ctx = UpdateVersionList( + (v, p) => _versions[v] = p, + addedProperties.ParsedVersion, + addedProperties); + + var searchIndexChangeType = UpsertToSearchIndex(ctx); + var hijackIndexChanges = UpsertToHijackIndex(ctx); + + return new LatestIndexChanges(searchIndexChangeType, hijackIndexChanges); + } + + private static SearchIndexChangeType DeleteFromSearchIndex(Context ctx) + { + if (ctx.NewLatest == null) + { + // If there is no longer a latest version, the search document should be deleted. + return SearchIndexChangeType.Delete; + } + + if (ctx.NewLatest != ctx.OldLatest) + { + // If the latest version changes due to deletion, this can only happen if the existing latest version + // was the one that was deleted. + Guard.Assert(ctx.OldProperties != null, "This version should have existed before."); + Guard.Assert(ctx.OldProperties.Listed, "The existing version should have been listed."); + Guard.Assert(ctx.OldLatest == ctx.ChangedVersion, "The existing latest should be the deleted version."); + return SearchIndexChangeType.DowngradeLatest; + } + + // It's possible some cases (such as removing a version that didn't exist in the first place) could be + // handled without any change to the search index. However, to support the reflow case, we update the + // version list to make sure it is consistent. + return SearchIndexChangeType.UpdateVersionList; + } + + private static IReadOnlyList DeleteFromHijackIndex(Context ctx) + { + var changes = new List(); + + // Delete the document for the deleted version. + changes.Add(HijackIndexChange.Delete(ctx.ChangedVersion)); + + // Update the latest status of the latest version, if there is one. + if (ctx.NewLatest != null) + { + Guard.Assert(ctx.ChangedVersion != ctx.NewLatest, "The deleted version should not be the new latest version."); + changes.Add(HijackIndexChange.SetLatestToTrue(ctx.NewLatest)); + } + + return changes; + } + + private static SearchIndexChangeType UpsertToSearchIndex(Context ctx) + { + if (ctx.NewLatest == null) + { + // If there is no longer a latest version, the search document should be deleted. + return SearchIndexChangeType.Delete; + } + + if (ctx.OldLatest == null && ctx.NewLatest != null) + { + // If there was no latest version before but now there is a latest version, then we have just added + // the only listed version. The new latest version is of course the version we are adding right now. + Guard.Assert(ctx.NewLatest == ctx.ChangedVersion, "The first latest version must be the added version."); + Guard.Assert(ctx.NewProperties.Listed, "The added version should be listed for the latest version to have changed."); + return SearchIndexChangeType.AddFirst; + } + + if (ctx.NewLatest == ctx.ChangedVersion) + { + // If the latest version has not changed or the latest version is the version we just added, + // then we need to update the existing metadata. This includes updating the latest version string. + return SearchIndexChangeType.UpdateLatest; + } + + if (ctx.NewLatest < ctx.OldLatest) + { + // If the new latest is a lower version than the old latest, this is a special case where we need to + // look up the old new latest's metadata. + Guard.Assert(ctx.NewLatest != ctx.ChangedVersion, "This case should already have been handled."); + Guard.Assert(ctx.OldLatest == ctx.ChangedVersion, "This case should already have been handled."); + Guard.Assert( + ctx.NewProperties == null || !ctx.NewProperties.Listed, + "A downgrade from an upserted version can only happen from an unlist or removing a non-applicable version."); + return SearchIndexChangeType.DowngradeLatest; + } + + // It's possible some cases (such as unlisting a version that didn't exist in the first place) could be + // handled without any change to the search index. However, to support the reflow case, we update the + // version list to make sure it is consistent. + return SearchIndexChangeType.UpdateVersionList; + } + + private static IReadOnlyList UpsertToHijackIndex(Context ctx) + { + var changes = new List(); + + // Update the metadata for the upserted version. + changes.Add(HijackIndexChange.UpdateMetadata(ctx.ChangedVersion)); + + // If the new latest is not the version that we are processing right now, explicitly set the current version + // to not be the latest. This supports the reflow scenario. + if (ctx.NewLatest != ctx.ChangedVersion) + { + changes.Add(HijackIndexChange.SetLatestToFalse(ctx.ChangedVersion)); + } + + // If the latest version has changed and the old latest version existed, mark that old latest version as + // no longer latest. + if (ctx.OldLatest != null + && ctx.OldLatest != ctx.NewLatest + && ctx.OldLatest != ctx.ChangedVersion) + { + changes.Add(HijackIndexChange.SetLatestToFalse(ctx.OldLatest)); + } + + // Always mark the new latest version as latest, even if it has not changed. This supports the reflow + // scenario. + if (ctx.NewLatest != null) + { + changes.Add(HijackIndexChange.SetLatestToTrue(ctx.NewLatest)); + } + + return changes; + } + + private Context UpdateVersionList( + Action update, + NuGetVersion version, + FilteredVersionProperties newProperties) + { + var oldLatest = _latestOrNull; + + _versions.TryGetValue(version, out var oldProperties); + + update(version, newProperties); + + _latestOrNull = CalculateLatest(); + + return new Context( + version, + oldProperties, + newProperties, + oldLatest, + _latestOrNull); + } + + private NuGetVersion CalculateLatest() + { + return _versions + .Reverse() + .Where(x => x.Value.Listed) + .Select(x => x.Value.ParsedVersion) + .FirstOrDefault(); + } + + private class Context + { + public Context( + NuGetVersion changedVersion, + FilteredVersionProperties oldProperties, + FilteredVersionProperties newProperties, + NuGetVersion oldLatest, + NuGetVersion newLatest) + { + ChangedVersion = changedVersion ?? throw new ArgumentNullException(nameof(changedVersion)); + OldProperties = oldProperties; + NewProperties = newProperties; + OldLatest = oldLatest; + NewLatest = newLatest; + } + + public NuGetVersion ChangedVersion { get; } + public FilteredVersionProperties OldProperties { get; } + public FilteredVersionProperties NewProperties { get; } + public NuGetVersion OldLatest { get; } + public NuGetVersion NewLatest { get; } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/FilteredVersionProperties.cs b/src/NuGet.Services.AzureSearch/VersionList/FilteredVersionProperties.cs new file mode 100644 index 000000000..12da12115 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/FilteredVersionProperties.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Version properties needed by . This is a subset of information needed by + /// . The extra information not contained in this class (such as + /// ) is only used for filtering. + /// + internal class FilteredVersionProperties + { + public FilteredVersionProperties(string fullVersion, NuGetVersion parsedVersion, bool listed) + { + FullVersion = fullVersion ?? throw new ArgumentNullException(nameof(fullVersion)); + ParsedVersion = parsedVersion ?? throw new ArgumentNullException(nameof(parsedVersion)); + Listed = listed; + } + + public FilteredVersionProperties(string fullVersion, bool listed) + { + if (fullVersion == null) + { + throw new ArgumentNullException(nameof(fullVersion)); + } + + ParsedVersion = NuGetVersion.Parse(fullVersion); + FullVersion = ParsedVersion.ToFullString(); + Listed = listed; + } + + public string FullVersion { get; } + public NuGetVersion ParsedVersion { get; } + public bool Listed { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/HijackDocumentChanges.cs b/src/NuGet.Services.AzureSearch/VersionList/HijackDocumentChanges.cs new file mode 100644 index 000000000..e0c82e2fc --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/HijackDocumentChanges.cs @@ -0,0 +1,71 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch +{ + public class HijackDocumentChanges + { + public HijackDocumentChanges( + bool delete, + bool updateMetadata, + bool latestStableSemVer1, + bool latestSemVer1, + bool latestStableSemVer2, + bool latestSemVer2) + { + if (delete && updateMetadata) + { + throw new ArgumentException("Deleting and updating a hijack document are mutually exclusive."); + } + + if (delete && (latestStableSemVer1 || latestSemVer1 || latestStableSemVer2 || latestSemVer2)) + { + throw new ArgumentException("Deleting a document is mutually exclusive with making that document that latest."); + } + + Delete = delete; + UpdateMetadata = updateMetadata; + LatestStableSemVer1 = latestStableSemVer1; + LatestSemVer1 = latestSemVer1; + LatestStableSemVer2 = latestStableSemVer2; + LatestSemVer2 = latestSemVer2; + } + + /// + /// Whether or not to delete the document. If this value is true, all other properties (aside from + /// ) are ignored. + /// + public bool Delete { get; } + + /// + /// Whether or not to update the metadata of this version. + /// + public bool UpdateMetadata { get; } + + /// + /// Whether or not this version is the latest version, excluding prerelease versions and SemVer 2.0.0 versions. + /// This is associated with . + /// + public bool LatestStableSemVer1 { get; } + + /// + /// Whether or not this version is the latest version, excluding SemVer 2.0.0 versions. This is associated with + /// . + /// + public bool LatestSemVer1 { get; } + + /// + /// Whether or not this version is the latest version, excluding prerelease versions. This is associated with + /// . + /// + public bool LatestStableSemVer2 { get; } + + /// + /// Whether or not this version is the latest version. This is associated with + /// . + /// + public bool LatestSemVer2 { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/HijackIndexChange.cs b/src/NuGet.Services.AzureSearch/VersionList/HijackIndexChange.cs new file mode 100644 index 000000000..1dda0dae9 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/HijackIndexChange.cs @@ -0,0 +1,71 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch +{ + public class HijackIndexChange : IEquatable + { + private HijackIndexChange(NuGetVersion version, HijackIndexChangeType type) + { + Version = version ?? throw new ArgumentNullException(nameof(version)); + Type = type; + } + + public static HijackIndexChange UpdateMetadata(NuGetVersion version) + { + return new HijackIndexChange(version, HijackIndexChangeType.UpdateMetadata); + } + + public static HijackIndexChange Delete(NuGetVersion version) + { + return new HijackIndexChange(version, HijackIndexChangeType.Delete); + } + + public static HijackIndexChange SetLatestToFalse(NuGetVersion version) + { + return new HijackIndexChange(version, HijackIndexChangeType.SetLatestToFalse); + } + + public static HijackIndexChange SetLatestToTrue(NuGetVersion version) + { + return new HijackIndexChange(version, HijackIndexChangeType.SetLatestToTrue); + } + + /// + /// The package version affected. + /// + public NuGetVersion Version { get; } + + /// + /// The type of the document change. + /// + public HijackIndexChangeType Type { get; } + + public override bool Equals(object obj) + { + return Equals(obj as HijackIndexChange); + } + + public bool Equals(HijackIndexChange change) + { + return change != null && + Version == change.Version && + Type == change.Type; + } + + /// + /// This was generated using Visual Studio. + /// + public override int GetHashCode() + { + var hashCode = 1834694972; + hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(Version); + hashCode = hashCode * -1521134295 + Type.GetHashCode(); + return hashCode; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/HijackIndexChangeType.cs b/src/NuGet.Services.AzureSearch/VersionList/HijackIndexChangeType.cs new file mode 100644 index 000000000..36216d43e --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/HijackIndexChangeType.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch +{ + public enum HijackIndexChangeType + { + /// + /// Update the metadata of the document. This is not a superset of + /// or . Suppose versions 1.0.0 and 2.0.0 are both listed. Then a catalog leaf + /// comes in unlisting version 2.0.0 (making it no longer the latest). In this case version 1.0.0 will be + /// changed using but will not have a change. + /// 2.0.0 will have both and changes. + /// + UpdateMetadata, + + /// + /// Delete the document. This change type is mutually exclusive with all other change types. + /// + Delete, + + /// + /// Set the latest property to true. This change type is mutually exclusive with + /// . + /// + SetLatestToTrue, + + /// + /// Set the latest property to false. this change type is mutually exclusive with + /// . + /// + SetLatestToFalse, + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/IVersionListDataClient.cs b/src/NuGet.Services.AzureSearch/VersionList/IVersionListDataClient.cs new file mode 100644 index 000000000..9ec25b882 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/IVersionListDataClient.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch +{ + /// + /// This interface allows the caller to read and write the Version List resource. This resource is an implementation + /// detail of the Azure Search ingestion pipeline and provides an ID -> Versions mapping. The information stored for + /// each version is enough to generate calculate what the latest version transition is per package ID, given the + /// caller's SemVer 2.0.0 filtering preference. In other words, each version has a "SemVer2" and "Listed" boolean + /// property. This version list should be read before pushing changes to the the Azure Search indexes and updated + /// after. See to see all information available per package ID. + /// + public interface IVersionListDataClient + { + Task> ReadAsync(string id); + + /// + /// Replace the version list of the provided package ID. May return false due to access condition (i.e. 412 + /// Precondition Failed in Azure Blob Storage). May throw exceptions unrelated to access condition failures. + /// False will be returned if another caller has modified the version list thus invalidating the access + /// condition. + /// + /// The package ID. + /// The data of the version list to be written. + /// The access condition for the write operation. + /// True if the access condition is accepted. False if the access condition fails. + Task TryReplaceAsync(string id, VersionListData data, IAccessCondition accessCondition); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/VersionList/IndexChanges.cs b/src/NuGet.Services.AzureSearch/VersionList/IndexChanges.cs new file mode 100644 index 000000000..530058dd6 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/IndexChanges.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Changes related to the search and hijack indexes for a specific ID. + /// + public class IndexChanges + { + public IndexChanges( + IReadOnlyDictionary search, + IReadOnlyDictionary hijack) + { + Search = search ?? throw new ArgumentNullException(nameof(search)); + Hijack = hijack ?? throw new ArgumentNullException(nameof(hijack)); + } + + public IReadOnlyDictionary Search { get; } + public IReadOnlyDictionary Hijack { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/LatestIndexChanges.cs b/src/NuGet.Services.AzureSearch/VersionList/LatestIndexChanges.cs new file mode 100644 index 000000000..ff8cf7ecd --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/LatestIndexChanges.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Changes related to the latest status in indexes. For the search index, this can be all of the metadata on the + /// document. For the hijack index, this only relates to the latest booleans. This type represents changes related + /// to a single value (i.e. one perspective of the indexes). + /// + public class LatestIndexChanges + { + public LatestIndexChanges(SearchIndexChangeType search, IReadOnlyList hijack) + { + Search = search; + Hijack = hijack ?? throw new ArgumentNullException(nameof(hijack)); + } + + public SearchIndexChangeType Search { get; } + public IReadOnlyList Hijack { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/LatestVersionInfo.cs b/src/NuGet.Services.AzureSearch/VersionList/LatestVersionInfo.cs new file mode 100644 index 000000000..3eb69ed52 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/LatestVersionInfo.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch +{ + public class LatestVersionInfo + { + public LatestVersionInfo(NuGetVersion parsedVersion, string fullVersion, string[] listedFullVersions) + { + ParsedVersion = parsedVersion ?? throw new ArgumentNullException(nameof(parsedVersion)); + FullVersion = fullVersion ?? throw new ArgumentNullException(fullVersion); + ListedFullVersions = listedFullVersions ?? throw new ArgumentNullException(nameof(listedFullVersions)); + } + + public NuGetVersion ParsedVersion { get; } + public string FullVersion { get; } + public string[] ListedFullVersions { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/MutableHijackDocumentChanges.cs b/src/NuGet.Services.AzureSearch/VersionList/MutableHijackDocumentChanges.cs new file mode 100644 index 000000000..eae91552c --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/MutableHijackDocumentChanges.cs @@ -0,0 +1,188 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; + +namespace NuGet.Services.AzureSearch +{ + /// + /// A mutable version of . The booleans in this class are nullable so that we can + /// track the "undetermined" state. Once the booleans are set to true or false, they cannot be changed again. This + /// helps protect against bugs in the code that calls + /// in an inconsistent manner. Technically, the + /// and do not need to be nullable because there is no code path + /// that can set them explicitly to false. However it's better to be consistent with the latest booleans and employ + /// the same strategy. Null booleans can be assumed by the caller to be false. + /// + internal class MutableHijackDocumentChanges : IEquatable + { + public MutableHijackDocumentChanges() + { + } + + public MutableHijackDocumentChanges( + bool? delete, + bool? updateMetadata, + bool? latestStableSemVer1, + bool? latestSemVer1, + bool? latestStableSemVer2, + bool? latestSemVer2) + { + Delete = delete; + UpdateMetadata = updateMetadata; + LatestStableSemVer1 = latestStableSemVer1; + LatestSemVer1 = latestSemVer1; + LatestStableSemVer2 = latestStableSemVer2; + LatestSemVer2 = latestSemVer2; + } + + public bool? Delete { get; private set; } + public bool? UpdateMetadata { get; private set; } + public bool? LatestStableSemVer1 { get; private set; } + public bool? LatestSemVer1 { get; private set; } + public bool? LatestStableSemVer2 { get; private set; } + public bool? LatestSemVer2 { get; private set; } + + public void ApplyChange(SearchFilters searchFilters, HijackIndexChangeType changeType) + { + bool latest; + switch (changeType) + { + case HijackIndexChangeType.Delete: + Guard.Assert( + Delete != false, + "The hijack document has already been set to not delete."); + Guard.Assert( + UpdateMetadata != true, + "The hijack document has already been set to update metadata."); + Delete = true; + foreach (var eachSearchFilters in DocumentUtilities.AllSearchFilters) + { + SetLatest(eachSearchFilters, latest: null); + } + return; + case HijackIndexChangeType.UpdateMetadata: + Guard.Assert( + UpdateMetadata != false, + "The hijack document has already been set to not update metadata."); + Guard.Assert( + Delete != true, + "The hijack document has already been set to delete so metadata can't be updated."); + UpdateMetadata = true; + return; + case HijackIndexChangeType.SetLatestToFalse: + latest = false; + break; + case HijackIndexChangeType.SetLatestToTrue: + latest = true; + break; + default: + throw new NotImplementedException($"The hijack index change type '{changeType}' is not supported."); + } + + Guard.Assert( + Delete != true, + "The hijack document has already been set to delete so the latest value can't be updated."); + SetLatest(searchFilters, latest); + } + + public bool? GetLatest(SearchFilters searchFilters) + { + switch (searchFilters) + { + case SearchFilters.Default: + return LatestStableSemVer1; + case SearchFilters.IncludePrerelease: + return LatestSemVer1; + case SearchFilters.IncludeSemVer2: + return LatestStableSemVer2; + case SearchFilters.IncludePrereleaseAndSemVer2: + return LatestSemVer2; + default: + throw new NotImplementedException($"The search filters '{searchFilters}' is not supported."); + } + } + + private void SetLatest(SearchFilters searchFilters, bool? latest) + { + switch (searchFilters) + { + case SearchFilters.Default: + LatestStableSemVer1 = latest; + break; + case SearchFilters.IncludePrerelease: + LatestSemVer1 = latest; + break; + case SearchFilters.IncludeSemVer2: + LatestStableSemVer2 = latest; + break; + case SearchFilters.IncludePrereleaseAndSemVer2: + LatestSemVer2 = latest; + break; + default: + throw new NotImplementedException($"The search filters '{searchFilters}' is not supported."); + } + } + + public HijackDocumentChanges Solidify() + { + return new HijackDocumentChanges( + Delete ?? false, + UpdateMetadata ?? false, + LatestStableSemVer1 ?? false, + LatestSemVer1 ?? false, + LatestStableSemVer2 ?? false, + LatestSemVer2 ?? false); + } + + public override bool Equals(object obj) + { + return Equals(obj as MutableHijackDocumentChanges); + } + + /// + /// This was generated using Visual Studio. + /// + public bool Equals(MutableHijackDocumentChanges document) + { + return document != null && + Delete == document.Delete && + UpdateMetadata == document.UpdateMetadata && + EqualityComparer.Default.Equals(LatestStableSemVer1, document.LatestStableSemVer1) && + EqualityComparer.Default.Equals(LatestSemVer1, document.LatestSemVer1) && + EqualityComparer.Default.Equals(LatestStableSemVer2, document.LatestStableSemVer2) && + EqualityComparer.Default.Equals(LatestSemVer2, document.LatestSemVer2); + } + + public static bool operator ==(MutableHijackDocumentChanges a, MutableHijackDocumentChanges b) + { + if (ReferenceEquals(a, null)) + { + return ReferenceEquals(b, null); + } + + return a.Equals(b); + } + + public static bool operator !=(MutableHijackDocumentChanges a, MutableHijackDocumentChanges b) + { + return !(a == b); + } + + /// + /// This was generated using Visual Studio. + /// + public override int GetHashCode() + { + var hashCode = -1628679267; + hashCode = hashCode * -1521134295 + Delete.GetHashCode(); + hashCode = hashCode * -1521134295 + UpdateMetadata.GetHashCode(); + hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(LatestStableSemVer1); + hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(LatestSemVer1); + hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(LatestStableSemVer2); + hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(LatestSemVer2); + return hashCode; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/MutableIndexChanges.cs b/src/NuGet.Services.AzureSearch/VersionList/MutableIndexChanges.cs new file mode 100644 index 000000000..92bcb0630 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/MutableIndexChanges.cs @@ -0,0 +1,241 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using NuGet.Versioning; +using SICT = NuGet.Services.AzureSearch.SearchIndexChangeType; + +namespace NuGet.Services.AzureSearch +{ + /// + /// A mutable version of . + /// + internal class MutableIndexChanges + { + /// + /// This is a dictionary where the key is the state transition. The value of the dictionary is the resulting + /// state from that transition. Remember that versions are processed in descending version order so some state + /// transition should not be possible. + /// + private static readonly IReadOnlyDictionary AcceptableTransitions + = new Dictionary + { + // Example: add an initial, listed version then add a lower, listed version + { new StateTransition(SICT.AddFirst, SICT.UpdateVersionList), SICT.AddFirst }, + + // Example: add an initial, unlisted version then add a lower, listed version + { new StateTransition(SICT.Delete, SICT.AddFirst), SICT.AddFirst }, + + // Example: add a new latest, listed version then add a lower, listed version + { new StateTransition(SICT.UpdateLatest, SICT.UpdateVersionList), SICT.UpdateLatest }, + + // Example: delete a non-existent version then unlist the latest version + { new StateTransition(SICT.UpdateVersionList, SICT.Delete), SICT.Delete }, + + // Example: unlist an already unlisted higher version then unlist the latest version + { new StateTransition(SICT.UpdateVersionList, SICT.DowngradeLatest), SICT.DowngradeLatest }, + + // Example: unlist an already unlisted higher version then add a new latest version + { new StateTransition(SICT.UpdateVersionList, SICT.UpdateLatest), SICT.UpdateLatest }, + + // Example: unlist the latest version then add a new latest version + { new StateTransition(SICT.DowngradeLatest, SICT.UpdateLatest), SICT.UpdateLatest }, + + // Example: unlist the latest version then add a new non-latest version + { new StateTransition(SICT.DowngradeLatest, SICT.UpdateVersionList), SICT.DowngradeLatest }, + + // Example: delete the latest version then delete the last latest version + { new StateTransition(SICT.DowngradeLatest, SICT.Delete), SICT.Delete }, + }; + + public MutableIndexChanges() + { + SearchChanges = new Dictionary(); + HijackChanges = new Dictionary>>(); + HijackDocuments = new Dictionary(); + } + + public MutableIndexChanges( + Dictionary search, + Dictionary>> hijack) + { + SearchChanges = search ?? throw new ArgumentNullException(nameof(search)); + HijackChanges = hijack ?? throw new ArgumentNullException(nameof(hijack)); + HijackDocuments = hijack.ToDictionary( + x => x.Key, + x => InitializeHijackDocumentChanges(x.Value)); + } + + private static MutableHijackDocumentChanges InitializeHijackDocumentChanges( + IEnumerable> changes) + { + var document = new MutableHijackDocumentChanges(); + foreach (var change in changes) + { + document.ApplyChange(change.Key, change.Value); + } + + return document; + } + + public Dictionary SearchChanges { get; } + private Dictionary>> HijackChanges { get; } + + /// + /// Keep track of the hijack document as we merge multiple . This allows + /// us to detect consistency problems are quickly as possible. + /// + public Dictionary HijackDocuments { get; } + + public static MutableIndexChanges FromLatestIndexChanges( + IReadOnlyDictionary latestIndexChanges) + { + // Take the search index changes as-is. + var search = latestIndexChanges.ToDictionary(x => x.Key, x => x.Value.Search); + + // Group hijack index changes by version. + var hijack = latestIndexChanges + .SelectMany(pair => pair + .Value + .Hijack + .Select(change => new { SearchFilters = pair.Key, change.Type, change.Version })) + .GroupBy(x => x.Version) + .ToDictionary( + x => x.Key, + x => x.Select(y => KeyValuePair.Create(y.SearchFilters, y.Type)).ToList()); + + return new MutableIndexChanges(search, hijack); + } + + public void Merge(MutableIndexChanges added) + { + if (added == null) + { + throw new ArgumentNullException(nameof(added)); + } + + foreach (var pair in added.SearchChanges) + { + MergeSearchIndexChanges(pair.Key, pair.Value); + } + + foreach (var pair in added.HijackChanges) + { + MergeHijackIndexChanges(pair.Key, pair.Value); + } + + // Verify that there are not multiple latest versions per search filter. + foreach (var searchFilters in DocumentUtilities.AllSearchFilters) + { + var latest = HijackDocuments + .Where(x => x.Value.GetLatest(searchFilters).GetValueOrDefault(false)) + .Select(x => x.Key.ToFullString()) + .ToList(); + Guard.Assert( + latest.Count <= 1, + $"There are {latest.Count} versions set to be latest on search filter {searchFilters}: {string.Join(", ", latest)}"); + } + } + + private void MergeSearchIndexChanges(SearchFilters searchFilters, SICT addedType) + { + if (!SearchChanges.TryGetValue(searchFilters, out var existingType)) + { + SearchChanges[searchFilters] = addedType; + return; + } + + // If the search index change type is the same, move on. + if (existingType == addedType) + { + return; + } + + var transition = new StateTransition(existingType, addedType); + if (AcceptableTransitions.TryGetValue(transition, out var result)) + { + SearchChanges[searchFilters] = result; + return; + } + + Guard.Fail($"A {existingType} search index change cannot be replaced with {addedType}."); + } + + private void MergeHijackIndexChanges( + NuGetVersion version, + List> addedChanges) + { + // If the version does not yet exist, add it and move on. + if (!HijackChanges.TryGetValue(version, out var existingChanges)) + { + HijackDocuments.Add(version, InitializeHijackDocumentChanges(addedChanges)); + HijackChanges.Add(version, addedChanges); + } + else + { + var document = HijackDocuments[version]; + foreach (var change in addedChanges) + { + document.ApplyChange(change.Key, change.Value); + } + + existingChanges.AddRange(addedChanges); + } + } + + public IndexChanges Solidify() + { + // Verify that the running list of hijack changes is the same as the pre-computed hijack document. + Guard.Assert(HijackChanges.Count == HijackDocuments.Count, "The hijack document state has diverged."); + foreach (var pair in HijackChanges) + { + var expected = InitializeHijackDocumentChanges(pair.Value); + var actual = HijackDocuments[pair.Key]; + Guard.Assert( + expected == actual, + $"The hijack document for {pair.Key.ToFullString()} is different than the list of index changes."); + } + + return new IndexChanges( + SearchChanges.ToDictionary(x => x.Key, x => x.Value), + HijackDocuments.ToDictionary(x => x.Key, x => x.Value.Solidify())); + } + + private class StateTransition : IEquatable + { + public StateTransition(SICT existing, SICT added) + { + Existing = existing; + Added = added; + } + + public SICT Existing { get; } + public SICT Added { get; } + + public override bool Equals(object obj) + { + return Equals(obj as StateTransition); + } + + public bool Equals(StateTransition transition) + { + return transition != null && + Existing == transition.Existing && + Added == transition.Added; + } + + /// + /// This method was generated by Visual Studio. + /// + public override int GetHashCode() + { + var hashCode = -699697695; + hashCode = hashCode * -1521134295 + Existing.GetHashCode(); + hashCode = hashCode * -1521134295 + Added.GetHashCode(); + return hashCode; + } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/ResultAndAccessCondition.cs b/src/NuGet.Services.AzureSearch/VersionList/ResultAndAccessCondition.cs new file mode 100644 index 000000000..5f41f8782 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/ResultAndAccessCondition.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGetGallery; + +namespace NuGet.Services.AzureSearch +{ + /// + /// Contains a result and the access condition used to modify the result in storage. + /// + /// The type of the result. + public class ResultAndAccessCondition + { + public ResultAndAccessCondition(T result, IAccessCondition accessCondition) + { + Result = result; + AccessCondition = accessCondition; + } + + public T Result { get; } + public IAccessCondition AccessCondition { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/SearchFilters.cs b/src/NuGet.Services.AzureSearch/VersionList/SearchFilters.cs new file mode 100644 index 000000000..de3da04ba --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/SearchFilters.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.AzureSearch +{ + [Flags] + public enum SearchFilters + { + /// + /// Exclude SemVer 2.0.0 and prerelease packages. + /// + Default = 0, + + /// + /// Include packages that have prerelease versions. Note that a package's dependency version ranges do not + /// affect the prerelease status of the package. This is in contrast of . + /// + IncludePrerelease = 1 << 0, + + /// + /// Include SemVer 2.0.0 packages. Note that SemVer 2.0.0 dependency version ranges make a package into a SemVer + /// 2.0.0 even if the package's own version string is SemVer 1.0.0. + /// + IncludeSemVer2 = 1 << 1, + + /// + /// Include package that have prerelease versions and include SemVer 2.0.0 packages. + /// + IncludePrereleaseAndSemVer2 = IncludePrerelease | IncludeSemVer2, + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/SearchIndexChangeType.cs b/src/NuGet.Services.AzureSearch/VersionList/SearchIndexChangeType.cs new file mode 100644 index 000000000..dd487ebff --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/SearchIndexChangeType.cs @@ -0,0 +1,38 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch +{ + public enum SearchIndexChangeType + { + /// + /// The only change necessary is updating the version list and "is latest" flags. Some potentially no-op cases + /// also fall into this category to support reflowing. + /// + UpdateVersionList, + + /// + /// The latest version has been deleted or unlisted, so we need to replace the latest version metadata with an + /// older version's metadata. + /// + DowngradeLatest, + + /// + /// Update the latest version's metadata and version list. This represents both updating the metadata of the + /// existing latest version and changing the latest version and metadata to a later version. + /// + UpdateLatest, + + /// + /// The first listed version has been added, so we need to fetch owner information in addition to replacing + /// metadata and version list. + /// + AddFirst, + + /// + /// The last version was unlisted or deleted, so we need to delete the document from the index. Some + /// potentially no-op cases also fall into this category to support reflowing. + /// + Delete, + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/SemVerOrderedDictionaryJsonConverter.cs b/src/NuGet.Services.AzureSearch/VersionList/SemVerOrderedDictionaryJsonConverter.cs new file mode 100644 index 000000000..fb5384042 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/SemVerOrderedDictionaryJsonConverter.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Newtonsoft.Json; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch +{ + /// + /// This is not strictly necessary but has proven useful for debugging the JSON blobs generated. + /// + public class SemVerOrderedDictionaryJsonConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return typeof(IReadOnlyDictionary).IsAssignableFrom(objectType); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + // Use default behavior for deserialization. + return serializer.Deserialize>(reader); + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + var dictionary = (IReadOnlyDictionary)value; + var pairs = dictionary + .OrderBy(x => + { + NuGetVersion.TryParse(x.Key, out var version); + return version; + }) + .ToList(); + + writer.WriteStartObject(); + + foreach (var pair in pairs) + { + writer.WritePropertyName(pair.Key); + serializer.Serialize(writer, pair.Value); + } + + writer.WriteEndObject(); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/VersionListChange.cs b/src/NuGet.Services.AzureSearch/VersionList/VersionListChange.cs new file mode 100644 index 000000000..9ee83299c --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/VersionListChange.cs @@ -0,0 +1,78 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch +{ + public class VersionListChange + { + private VersionListChange(bool isDelete, NuGetVersion parsedVersion, string fullVersion, VersionPropertiesData data) + { + IsDelete = isDelete; + ParsedVersion = parsedVersion ?? throw new ArgumentNullException(nameof(parsedVersion)); + FullVersion = fullVersion; + Data = data; + } + + public bool IsDelete { get; } + public NuGetVersion ParsedVersion { get; } + + /// + /// When is true, this value is null. + /// + public string FullVersion { get; } + + /// + /// When is true, this value is null. + /// + public VersionPropertiesData Data { get; } + + /// + /// Initialize a version list change representing a delete of the provided version. + /// + /// The version. This can be parsed from any form of the version. + /// The version list change. + public static VersionListChange Delete(NuGetVersion parsedVersion) + { + if (parsedVersion == null) + { + throw new ArgumentNullException(nameof(parsedVersion)); + } + + return new VersionListChange( + isDelete: true, + parsedVersion: parsedVersion, + fullVersion: null, + data: null); + } + + /// + /// Initialize a version list change representing an upsert of the provided version. + /// + /// The full version string or the original version string. + /// The properties relevent to the version list resource. + /// The version list change. + public static VersionListChange Upsert(string fullOrOriginalVersion, VersionPropertiesData data) + { + if (data == null) + { + throw new ArgumentNullException(nameof(data)); + } + + if (fullOrOriginalVersion == null) + { + throw new ArgumentNullException(nameof(fullOrOriginalVersion)); + } + + var parsedVersion = NuGetVersion.Parse(fullOrOriginalVersion); + var fullVersion = parsedVersion.ToFullString(); + return new VersionListChange( + isDelete: false, + parsedVersion: parsedVersion, + fullVersion: fullVersion, + data: data); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/VersionListData.cs b/src/NuGet.Services.AzureSearch/VersionList/VersionListData.cs new file mode 100644 index 000000000..dea0f1947 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/VersionListData.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch +{ + public class VersionListData + { + [JsonConstructor] + public VersionListData(Dictionary versionProperties) + { + VersionProperties = versionProperties ?? throw new ArgumentNullException(nameof(versionProperties)); + } + + /// + /// A dictionary of all versions currently available for this package. This includes listed and unlisted + /// versions. The key is the full version string (can include build metadata). + /// + [JsonConverter(typeof(SemVerOrderedDictionaryJsonConverter))] + public IReadOnlyDictionary VersionProperties { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/VersionListDataClient.cs b/src/NuGet.Services.AzureSearch/VersionList/VersionListDataClient.cs new file mode 100644 index 000000000..01bf18670 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/VersionListDataClient.cs @@ -0,0 +1,121 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Newtonsoft.Json; +using NuGetGallery; + +namespace NuGet.Services.AzureSearch +{ + public class VersionListDataClient : IVersionListDataClient + { + private static readonly JsonSerializer Serializer = JsonSerializer.Create(new JsonSerializerSettings + { + DefaultValueHandling = DefaultValueHandling.Ignore, // Prefer more terse serialization. + Formatting = Formatting.Indented, // Negligable performance impact but much more readable. + }); + + private readonly ICloudBlobClient _cloudBlobClient; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + private readonly Lazy _lazyContainer; + + public VersionListDataClient( + ICloudBlobClient cloudBlobClient, + IOptionsSnapshot options, + ILogger logger) + { + _cloudBlobClient = cloudBlobClient ?? throw new ArgumentNullException(nameof(cloudBlobClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + _lazyContainer = new Lazy( + () => _cloudBlobClient.GetContainerReference(_options.Value.StorageContainer)); + } + + private ICloudBlobContainer Container => _lazyContainer.Value; + + public async Task> ReadAsync(string id) + { + var blobReference = Container.GetBlobReference(GetFileName(id)); + + _logger.LogInformation("Reading the version list for package ID {PackageId}.", id); + + VersionListData data; + IAccessCondition accessCondition; + try + { + using (var stream = await blobReference.OpenReadAsync(AccessCondition.GenerateEmptyCondition())) + using (var streamReader = new StreamReader(stream)) + using (var jsonTextReader = new JsonTextReader(streamReader)) + { + data = Serializer.Deserialize(jsonTextReader); + } + + accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blobReference.ETag); + } + catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound) + { + data = new VersionListData(new Dictionary()); + accessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition(); + } + + return new ResultAndAccessCondition(data, accessCondition); + } + + public async Task TryReplaceAsync(string id, VersionListData data, IAccessCondition accessCondition) + { + using (var stream = new MemoryStream()) + { + using (var streamWriter = new StreamWriter( + stream, + encoding: new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true), + bufferSize: 1024, + leaveOpen: true)) + using (var jsonTextWriter = new JsonTextWriter(streamWriter)) + { + Serializer.Serialize(jsonTextWriter, data); + } + + stream.Position = 0; + + _logger.LogInformation("Replacing the version list for package ID {PackageId}.", id); + + var mappedAccessCondition = new AccessCondition + { + IfNoneMatchETag = accessCondition.IfNoneMatchETag, + IfMatchETag = accessCondition.IfMatchETag, + }; + + var blobReference = Container.GetBlobReference(GetFileName(id)); + blobReference.Properties.ContentType = "application/json"; + + try + { + await blobReference.UploadFromStreamAsync( + stream, + mappedAccessCondition); + return true; + } + catch (StorageException ex) when (ex.IsPreconditionFailedException()) + { + _logger.LogWarning(ex, "Replacing the version list for {Id} failed due to access condition.", id); + return false; + } + } + } + + private string GetFileName(string id) + { + return $"{_options.Value.NormalizeStoragePath()}version-lists/{id.ToLowerInvariant()}.json"; + } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/VersionLists.cs b/src/NuGet.Services.AzureSearch/VersionList/VersionLists.cs new file mode 100644 index 000000000..00dbb6b6b --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/VersionLists.cs @@ -0,0 +1,203 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch +{ + /// + /// A container for all versions lists. There is one version list tracked per combination of + /// flags. + /// + public class VersionLists + { + private static readonly IReadOnlyDictionary> SearchFilterPredicates + = new Dictionary> + { + { + SearchFilters.Default, + p => !p.ParsedVersion.IsPrerelease && !p.Data.SemVer2 + }, + { + SearchFilters.IncludePrerelease, + p => !p.Data.SemVer2 + }, + { + SearchFilters.IncludeSemVer2, + p => !p.ParsedVersion.IsPrerelease + }, + { + SearchFilters.IncludePrereleaseAndSemVer2, + p => true + }, + }; + + internal readonly Dictionary _versionLists; + internal readonly SortedDictionary> _versionProperties; + + public VersionLists(VersionListData data) + { + var allVersions = data + .VersionProperties + .Select(p => new VersionProperties(p.Key, p.Value)) + .OrderBy(x => x.ParsedVersion) + .ToList(); + + _versionProperties = new SortedDictionary>(); + foreach (var version in allVersions) + { + _versionProperties.Add(version.ParsedVersion, KeyValuePair.Create(version.FullVersion, version.Data)); + } + + _versionLists = new Dictionary(); + foreach (var pair in SearchFilterPredicates) + { + var searchFilter = pair.Key; + var predicate = pair.Value; + var listState = new FilteredVersionList(allVersions + .Where(predicate) + .Select(x => x.Filtered)); + _versionLists.Add(searchFilter, listState); + } + } + + public LatestVersionInfo GetLatestVersionInfoOrNull(SearchFilters searchFilters) + { + if (!_versionLists.TryGetValue(searchFilters, out var listState)) + { + return null; + } + + return listState.GetLatestVersionInfo(); + } + + public VersionListData GetVersionListData() + { + return new VersionListData(_versionProperties.Values.ToDictionary(x => x.Key, x => x.Value)); + } + + public IndexChanges ApplyChanges(IEnumerable changes) + { + return ApplyChangesInternal(changes).Solidify(); + } + + internal MutableIndexChanges ApplyChangesInternal(IEnumerable changes) + { + var mutableChanges = new MutableIndexChanges(); + + // Process the changes in descending order. + var sortedChanges = changes + .OrderByDescending(x => x.ParsedVersion) + .ToList(); + + // Verify that there is only one change per version. + var versionsWithMultipleChanges = sortedChanges + .GroupBy(x => x.ParsedVersion) + .OrderBy(x => x.Key) + .Select(x => KeyValuePair.Create(x.Key.ToFullString(), x.Count())) + .Where(x => x.Value > 1) + .ToList(); + if (versionsWithMultipleChanges.Any()) + { + throw new ArgumentException( + $"There are multiple changes for the following version(s): " + + string.Join(", ", versionsWithMultipleChanges.Select(x => $"{x.Key} ({x.Value} changes)")), + nameof(changes)); + } + + foreach (var change in sortedChanges) + { + MutableIndexChanges versionChanges; + if (!change.IsDelete) + { + versionChanges = Upsert(change.FullVersion, change.ParsedVersion, change.Data); + } + else + { + versionChanges = Delete(change.ParsedVersion); + } + + mutableChanges.Merge(versionChanges); + } + + // Verify that we are updating the metadata of all non-delete changes. + foreach (var change in changes) + { + Guard.Assert( + mutableChanges.HijackDocuments.ContainsKey(change.ParsedVersion), + $"The should be a hijack document for each changed version. Version {change.FullVersion} does " + + "not have a hijack document."); + + if (!change.IsDelete) + { + Guard.Assert( + mutableChanges.HijackDocuments[change.ParsedVersion].UpdateMetadata == true, + $"The metadata of version {change.FullVersion} should be updated."); + } + } + + return mutableChanges; + } + + internal MutableIndexChanges Upsert(string fullOrOriginalVersion, VersionPropertiesData data) + { + var parsedVersion = NuGetVersion.Parse(fullOrOriginalVersion); + return Upsert(parsedVersion.ToFullString(), parsedVersion, data); + } + + private MutableIndexChanges Upsert( + string fullVersion, + NuGetVersion parsedVersion, + VersionPropertiesData data) + { + var added = new VersionProperties(fullVersion, parsedVersion, data); + _versionProperties[added.ParsedVersion] = KeyValuePair.Create(added.FullVersion, data); + + // Detect changes related to the latest versions, per search filter. + var output = new Dictionary(); + foreach (var pair in _versionLists) + { + var searchFilter = pair.Key; + var listState = pair.Value; + var predicate = SearchFilterPredicates[searchFilter]; + + LatestIndexChanges latestIndexChanges; + if (predicate(added)) + { + latestIndexChanges = listState.Upsert(added.Filtered); + } + else + { + latestIndexChanges = listState.Remove(added.ParsedVersion); + } + + output[searchFilter] = latestIndexChanges; + } + + return MutableIndexChanges.FromLatestIndexChanges(output); + } + + internal MutableIndexChanges Delete(NuGetVersion parsedVersion) + { + _versionProperties.Remove(parsedVersion); + + // Detect changes related to the latest versions, per search filter. + var output = new Dictionary(); + foreach (var pair in _versionLists) + { + var searchFilter = pair.Key; + var listState = pair.Value; + + // We can execute this on all lists, no matter the search filter predicate because removing a version + // that was never added will result in recalculating the version list, which supports the reflow + // scenario. + output[searchFilter] = listState.Delete(parsedVersion); + } + + return MutableIndexChanges.FromLatestIndexChanges(output); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/VersionProperties.cs b/src/NuGet.Services.AzureSearch/VersionList/VersionProperties.cs new file mode 100644 index 000000000..e2118b80d --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/VersionProperties.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch +{ + /// + /// All properties related to a specific version. This type exists to avoid parsing the + /// over and over to get the . + /// + internal class VersionProperties + { + public VersionProperties(string fullVersion, NuGetVersion parsedVersion, VersionPropertiesData data) + { + Data = data ?? throw new ArgumentNullException(nameof(data)); + Filtered = new FilteredVersionProperties(fullVersion, parsedVersion, Data.Listed); + } + + public VersionProperties(string fullOrOriginalVersion, VersionPropertiesData data) + { + Data = data ?? throw new ArgumentNullException(nameof(data)); + Filtered = new FilteredVersionProperties(fullOrOriginalVersion, Data.Listed); + } + + public string FullVersion => Filtered.FullVersion; + public NuGetVersion ParsedVersion => Filtered.ParsedVersion; + + public VersionPropertiesData Data { get; } + public FilteredVersionProperties Filtered { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/VersionList/VersionPropertiesData.cs b/src/NuGet.Services.AzureSearch/VersionList/VersionPropertiesData.cs new file mode 100644 index 000000000..ebb6aacf9 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/VersionList/VersionPropertiesData.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch +{ + public class VersionPropertiesData + { + [JsonConstructor] + public VersionPropertiesData(bool listed, bool semVer2) + { + Listed = listed; + SemVer2 = semVer2; + } + + public bool Listed { get; } + public bool SemVer2 { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/WebExceptionRetryDelegatingHandler.cs b/src/NuGet.Services.AzureSearch/WebExceptionRetryDelegatingHandler.cs new file mode 100644 index 000000000..2fd237890 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/WebExceptionRetryDelegatingHandler.cs @@ -0,0 +1,54 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.AzureSearch +{ + public class WebExceptionRetryDelegatingHandler : DelegatingHandler + { + private static readonly HashSet _transientWebExceptionStatuses = new HashSet(new[] + { + WebExceptionStatus.ConnectFailure, // Unable to connect to the remote server + WebExceptionStatus.ConnectionClosed, // The underlying connection was closed + WebExceptionStatus.KeepAliveFailure, // A connection that was expected to be kept alive was closed by the server + WebExceptionStatus.ReceiveFailure, // An unexpected error occurred on a receive + }); + + private readonly ILogger _logger; + + public WebExceptionRetryDelegatingHandler(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + try + { + return await base.SendAsync(request, cancellationToken); + } + catch (Exception ex) when (ex is HttpRequestException hre && hre.InnerException is WebException we) + { + if (_transientWebExceptionStatuses.Contains(we.Status)) + { + // Retry only a single time since some of these transient exceptions take a while (~20 seconds) to be + // thrown and we don't want to make the user wait too long even to see a failure. + _logger.LogWarning(ex, "Transient web exception encountered, status {Status}. Attempting a single retry.", we.Status); + return await base.SendAsync(request, cancellationToken); + } + else + { + _logger.LogError(ex, "Non-transient web exception encountered, status {Status}.", we.Status); + throw; + } + } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Wrappers/DocumentsOperationsWrapper.cs b/src/NuGet.Services.AzureSearch/Wrappers/DocumentsOperationsWrapper.cs new file mode 100644 index 000000000..f3da8fd57 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Wrappers/DocumentsOperationsWrapper.cs @@ -0,0 +1,110 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net; +using System.Threading.Tasks; +using Microsoft.Azure.Search; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Logging; +using Microsoft.Rest.Azure; +using NuGet.Services.AzureSearch.SearchService; + +namespace NuGet.Services.AzureSearch.Wrappers +{ + public class DocumentsOperationsWrapper : IDocumentsOperationsWrapper + { + private readonly IDocumentsOperations _inner; + private readonly ILogger _logger; + + public DocumentsOperationsWrapper( + IDocumentsOperations inner, + ILogger logger) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task IndexAsync(IndexBatch batch) where T : class + { + return await _inner.IndexAsync(batch); + } + + public async Task GetOrNullAsync(string key) where T : class + { + return await RetryAsync( + nameof(GetOrNullAsync) + "", + () => _inner.GetAsync(key), + allow404: true); + } + + public async Task SearchAsync(string searchText, SearchParameters searchParameters) + { + return await RetryAsync( + nameof(SearchAsync), + () => _inner.SearchAsync(searchText, searchParameters), + allow404: false); + } + + public async Task> SearchAsync(string searchText, SearchParameters searchParameters) where T : class + { + return await RetryAsync( + nameof(SearchAsync) + "", + () => _inner.SearchAsync(searchText, searchParameters), + allow404: false); + } + + public async Task CountAsync() + { + return await RetryAsync( + nameof(CountAsync), + async () => await _inner.CountAsync(), + allow404: false); + } + + private async Task RetryAsync(string name, Func> actAsync, bool allow404) + { + const int maxAttempts = 3; + int currentAttempt = 0; + while (true) + { + currentAttempt++; + try + { + return await actAsync(); + } + catch (NullReferenceException ex) + { + if (currentAttempt < maxAttempts) + { + _logger.LogWarning( + "Operation {Name} failed attempt {CurrentAttempt} with a null reference exception. " + + "Retrying.", + name, + currentAttempt); + continue; + } + + // There is a bug where an inner RetryDelegatingHandler fails with NullReferenceException when the + // service is under load and attempting a search. Throw a clearer exception for now so we can + // understand the frequency in logs. + // https://github.com/Azure/azure-sdk-for-net/issues/3224 + // https://github.com/NuGet/Engineering/issues/2511 + throw new AzureSearchException("The search query failed due to Azure/azure-sdk-for-net#3224.", ex); + } + catch (CloudException ex) when (allow404 && ex.Response?.StatusCode == HttpStatusCode.NotFound) + { + return default(T); + } + catch (CloudException ex) when (ex.Response?.StatusCode == HttpStatusCode.BadRequest) + { + throw new InvalidSearchRequestException("The provided query is invalid.", ex); + } + catch (Exception ex) + { + throw new AzureSearchException($"Operation {name} failed.", ex); + } + } + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Wrappers/IDocumentsOperationsWrapper.cs b/src/NuGet.Services.AzureSearch/Wrappers/IDocumentsOperationsWrapper.cs new file mode 100644 index 000000000..4311e276b --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Wrappers/IDocumentsOperationsWrapper.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch.Wrappers +{ + public interface IDocumentsOperationsWrapper + { + Task IndexAsync(IndexBatch batch) where T : class; + Task GetOrNullAsync(string key) where T : class; + Task SearchAsync( + string searchText, + SearchParameters searchParameters); + Task> SearchAsync( + string searchText, + SearchParameters searchParameters) where T : class; + Task CountAsync(); + } +} diff --git a/src/NuGet.Services.AzureSearch/Wrappers/IIndexesOperationsWrapper.cs b/src/NuGet.Services.AzureSearch/Wrappers/IIndexesOperationsWrapper.cs new file mode 100644 index 000000000..78313a526 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Wrappers/IIndexesOperationsWrapper.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch.Wrappers +{ + public interface IIndexesOperationsWrapper + { + ISearchIndexClientWrapper GetClient(string indexName); + Task CreateAsync(Index index); + Task DeleteAsync(string indexName); + Task ExistsAsync(string indexName); + } +} diff --git a/src/NuGet.Services.AzureSearch/Wrappers/ISearchIndexClientWrapper.cs b/src/NuGet.Services.AzureSearch/Wrappers/ISearchIndexClientWrapper.cs new file mode 100644 index 000000000..9af07a24b --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Wrappers/ISearchIndexClientWrapper.cs @@ -0,0 +1,11 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.Wrappers +{ + public interface ISearchIndexClientWrapper + { + string IndexName { get; } + IDocumentsOperationsWrapper Documents { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Wrappers/ISearchServiceClientWrapper.cs b/src/NuGet.Services.AzureSearch/Wrappers/ISearchServiceClientWrapper.cs new file mode 100644 index 000000000..91352f140 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Wrappers/ISearchServiceClientWrapper.cs @@ -0,0 +1,10 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.AzureSearch.Wrappers +{ + public interface ISearchServiceClientWrapper + { + IIndexesOperationsWrapper Indexes { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Wrappers/ISystemTime.cs b/src/NuGet.Services.AzureSearch/Wrappers/ISystemTime.cs new file mode 100644 index 000000000..c30cb5f7f --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Wrappers/ISystemTime.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch.Wrappers +{ + /// + /// A wrapper that allows for unit tests related to system time. + /// + public interface ISystemTime + { + Task Delay(TimeSpan delay); + Task Delay(TimeSpan delay, CancellationToken token); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.AzureSearch/Wrappers/IndexesOperationsWrapper.cs b/src/NuGet.Services.AzureSearch/Wrappers/IndexesOperationsWrapper.cs new file mode 100644 index 000000000..af1a56bb8 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Wrappers/IndexesOperationsWrapper.cs @@ -0,0 +1,74 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.Azure.Search; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Logging; +using Microsoft.Rest.TransientFaultHandling; + +namespace NuGet.Services.AzureSearch.Wrappers +{ + public class IndexesOperationsWrapper : IIndexesOperationsWrapper + { + private readonly IIndexesOperations _inner; + private readonly DelegatingHandler[] _handlers; + private readonly RetryPolicy _retryPolicy; + private readonly ILogger _documentsOperationsLogger; + + public IndexesOperationsWrapper( + IIndexesOperations inner, + DelegatingHandler[] handlers, + RetryPolicy retryPolicy, + ILogger documentsOperationsLogger) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + _handlers = handlers ?? throw new ArgumentNullException(nameof(handlers)); + _retryPolicy = retryPolicy; + _documentsOperationsLogger = documentsOperationsLogger ?? throw new ArgumentNullException(nameof(documentsOperationsLogger)); + } + + /// + /// This is implemented in lieu of + /// because it allows the delegating handlers and retry policy to be specified. See: + /// https://github.com/Azure/azure-sdk-for-net/blob/96421089bc26198098f320ea50e0208e98376956/sdk/search/Microsoft.Azure.Search/src/IndexesGetClientExtensions.cs#L27-L41 + /// + public ISearchIndexClientWrapper GetClient(string indexName) + { + var searchIndexClient = new SearchIndexClient( + _inner.Client.SearchServiceName, + indexName, + _inner.Client.SearchCredentials, + _inner.Client.HttpMessageHandlers.OfType().SingleOrDefault(), + _handlers); + + searchIndexClient.SearchDnsSuffix = _inner.Client.SearchDnsSuffix; + searchIndexClient.HttpClient.Timeout = _inner.Client.HttpClient.Timeout; + + if (_retryPolicy != null) + { + searchIndexClient.SetRetryPolicy(_retryPolicy); + } + + return new SearchIndexClientWrapper(searchIndexClient, _documentsOperationsLogger); + } + + public async Task ExistsAsync(string indexName) + { + return await _inner.ExistsAsync(indexName); + } + + public async Task DeleteAsync(string indexName) + { + await _inner.DeleteAsync(indexName); + } + + public async Task CreateAsync(Index index) + { + return await _inner.CreateAsync(index); + } + } +} diff --git a/src/NuGet.Services.AzureSearch/Wrappers/SearchIndexClientWrapper.cs b/src/NuGet.Services.AzureSearch/Wrappers/SearchIndexClientWrapper.cs new file mode 100644 index 000000000..b90157598 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Wrappers/SearchIndexClientWrapper.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Azure.Search; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.AzureSearch.Wrappers +{ + public class SearchIndexClientWrapper : ISearchIndexClientWrapper + { + private readonly ISearchIndexClient _inner; + + public SearchIndexClientWrapper( + ISearchIndexClient inner, + ILogger documentsOperationsLogger) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + Documents = new DocumentsOperationsWrapper(_inner.Documents, documentsOperationsLogger); + } + + public string IndexName => _inner.IndexName; + public IDocumentsOperationsWrapper Documents { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Wrappers/SearchServiceClientWrapper.cs b/src/NuGet.Services.AzureSearch/Wrappers/SearchServiceClientWrapper.cs new file mode 100644 index 000000000..e926a4b53 --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Wrappers/SearchServiceClientWrapper.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using Microsoft.Azure.Search; +using Microsoft.Extensions.Logging; +using Microsoft.Rest.TransientFaultHandling; + +namespace NuGet.Services.AzureSearch.Wrappers +{ + public class SearchServiceClientWrapper : ISearchServiceClientWrapper + { + private readonly ISearchServiceClient _inner; + + public SearchServiceClientWrapper( + ISearchServiceClient inner, + DelegatingHandler[] handlers, + RetryPolicy retryPolicy, + ILogger documentsOperationsLogger) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + Indexes = new IndexesOperationsWrapper(_inner.Indexes, handlers, retryPolicy, documentsOperationsLogger); + } + + public IIndexesOperationsWrapper Indexes { get; } + } +} diff --git a/src/NuGet.Services.AzureSearch/Wrappers/SystemTime.cs b/src/NuGet.Services.AzureSearch/Wrappers/SystemTime.cs new file mode 100644 index 000000000..f3e8e054e --- /dev/null +++ b/src/NuGet.Services.AzureSearch/Wrappers/SystemTime.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.AzureSearch.Wrappers +{ + public class SystemTime : ISystemTime + { + public async Task Delay(TimeSpan delay) + { + await Task.Delay(delay); + } + + public async Task Delay(TimeSpan delay, CancellationToken token) + { + await Task.Delay(delay, token); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/NonhijackedV2HttpMessageHandler.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/NonhijackedV2HttpMessageHandler.cs new file mode 100644 index 000000000..83af417df --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/NonhijackedV2HttpMessageHandler.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// A that prevents queries to the V2 feed from being hijacked by the search service. + /// + public class NonhijackedV2HttpMessageHandler : DelegatingHandler + { + /// The to wrap. + public NonhijackedV2HttpMessageHandler(HttpMessageHandler inner) + : base(inner) + { + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + request.RequestUri = UriUtils.GetNonhijackableUri(request.RequestUri); + return base.SendAsync(request, cancellationToken); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationAlternatePackageMetadata.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationAlternatePackageMetadata.cs new file mode 100644 index 000000000..330b2abab --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationAlternatePackageMetadata.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Model +{ + public class PackageRegistrationAlternatePackageMetadata + { + public string Id { get; set; } + public string Range { get; set; } + + /// + /// Default constructor for JSON serialization purposes. + /// + public PackageRegistrationAlternatePackageMetadata() + { + } + + /// + /// Converts a into a format that can be directly compared to a . + /// + public PackageRegistrationAlternatePackageMetadata(PackageDeprecationItem deprecation) + { + Id = deprecation.AlternatePackageId; + Range = deprecation.AlternatePackageRange; + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationDeprecationMetadata.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationDeprecationMetadata.cs new file mode 100644 index 000000000..7d2a170c0 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationDeprecationMetadata.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Model +{ + public class PackageRegistrationDeprecationMetadata + { + public IEnumerable Reasons { get; set; } + public string Message { get; set; } + public PackageRegistrationAlternatePackageMetadata AlternatePackage { get; set; } + + /// + /// Default constructor for JSON serialization purposes. + /// + public PackageRegistrationDeprecationMetadata() + { + } + + /// + /// Converts a into a format that can be directly compared to a . + /// + public PackageRegistrationDeprecationMetadata(PackageDeprecationItem deprecation) + { + Reasons = deprecation.Reasons; + Message = deprecation.Message; + if (deprecation.AlternatePackageId != null || deprecation.AlternatePackageRange != null) + { + AlternatePackage = new PackageRegistrationAlternatePackageMetadata(deprecation); + } + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationIndexMetadata.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationIndexMetadata.cs new file mode 100644 index 000000000..8707cd36f --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationIndexMetadata.cs @@ -0,0 +1,51 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NuGet.Protocol; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring.Model; +using NuGet.Versioning; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// The metadata for a particular package in its registration index. + /// + public class PackageRegistrationIndexMetadata : PackageRegistrationLeafMetadata + { + public string Id { get; set; } + + [JsonConverter(typeof(NuGetVersionConverter))] + public NuGetVersion Version { get; set; } + + /// + /// In the database, this property is called "RequiresLicenseAcceptance" (notice the "s"). + /// + public bool RequireLicenseAcceptance { get; set; } + + public PackageRegistrationDeprecationMetadata Deprecation { get; set; } + + /// + /// Default constructor for JSON serialization purposes. + /// + public PackageRegistrationIndexMetadata() + { + } + + /// + /// Converts a into a format that can be directly compared to a . + /// + public PackageRegistrationIndexMetadata(FeedPackageDetails package) + : base(package) + { + Id = package.PackageId; + Version = NuGetVersion.Parse(package.PackageNormalizedVersion); + RequireLicenseAcceptance = package.RequiresLicenseAcceptance; + if (package.HasDeprecationInfo) + { + Deprecation = new PackageRegistrationDeprecationMetadata(package.DeprecationInfo); + } + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationLeafMetadata.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationLeafMetadata.cs new file mode 100644 index 000000000..f186529ee --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageRegistrationLeafMetadata.cs @@ -0,0 +1,35 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// The metadata for a particular package in its registration leaf. + /// See: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-leaf + /// + public class PackageRegistrationLeafMetadata + { + public bool Listed { get; set; } + + public DateTimeOffset? Published { get; set; } + + /// + /// Default constructor for JSON serialization purposes. + /// + public PackageRegistrationLeafMetadata() + { + } + + /// + /// Converts a into a format that can be directly compared to a . + /// + public PackageRegistrationLeafMetadata(FeedPackageDetails package) + { + Listed = PackageCatalogItem.GetListed(package.PublishedDate); + Published = package.PublishedDate; + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageTimestampMetadata.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageTimestampMetadata.cs new file mode 100644 index 000000000..5ef73fae0 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Model/PackageTimestampMetadata.cs @@ -0,0 +1,120 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NuGet.Protocol.Core.Types; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Represents the timestamp metadata for a single package in a package source. + /// + public class PackageTimestampMetadata : INuGetResource + { + public bool Exists { get; set; } + public DateTime? Created { get; set; } + public DateTime? LastEdited { get; set; } + public DateTime? Deleted { get; set; } + + /// + /// The most recent time the package was created, edited, or deleted in a package source. + /// + public DateTime? Last => new[] { Created, LastEdited, Deleted }.Max(); + + /// + /// Creates a that represents a package that exists on the package source. + /// + public static PackageTimestampMetadata CreateForExistingPackage(DateTime created, DateTime lastEdited) + { + return new PackageTimestampMetadata + { + Exists = true, + Created = created, + LastEdited = lastEdited, + Deleted = null + }; + } + + /// + /// Creates a that represents a package that is missing from the package source. + /// + public static PackageTimestampMetadata CreateForMissingPackage(DateTime? deleted) + { + return new PackageTimestampMetadata + { + Exists = false, + Created = null, + LastEdited = null, + Deleted = deleted + }; + } + + /// + /// In the past, the catalog entries referred to the CDN's host instead our DNS. + /// If we encounter one of these outdated hosts, we should hit the proper host instead. + /// + private static readonly IDictionary _catalogEntryUriHostMap = + new Dictionary + { + { "az635243.vo.msecnd.net", "apidev.nugettest.org" }, + { "az636225.vo.msecnd.net", "apiint.nugettest.org" } + }; + + public static async Task FromCatalogEntry( + CollectorHttpClient client, + CatalogIndexEntry catalogEntry) + { + var uri = catalogEntry.Uri; + if (_catalogEntryUriHostMap.TryGetValue(catalogEntry.Uri.Host, out var replacementHost)) + { + var builder = new UriBuilder(uri) + { + Host = replacementHost + }; + + uri = builder.Uri; + } + + var catalogLeaf = await client.GetJObjectAsync(uri); + + try + { + if (catalogEntry.IsDelete) + { + // On the catalog page for a delete, the published value is the timestamp the package was deleted from the audit records. + var deleted = catalogLeaf.GetValue("published").Value(); + + return CreateForMissingPackage(deleted.DateTime); + } + else + { + var created = catalogLeaf.GetValue("created").Value(); + var lastEdited = catalogLeaf.GetValue("lastEdited").Value(); + + return CreateForExistingPackage(created.DateTime, lastEdited.DateTime); + } + } + catch (Exception e) + { + throw new ArgumentException("Failed to create PackageTimestampMetadata from CatalogIndexEntry!", e); + } + } + + public static async Task FromCatalogEntries( + CollectorHttpClient client, + IEnumerable catalogEntries) + { + var packageTimestampMetadatas = + await Task.WhenAll(catalogEntries.Select(entry => FromCatalogEntry(client, entry))); + + return packageTimestampMetadatas + .Where(p => p != null) + .OrderByDescending(p => p.Last) + .First(); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/AuditingStoragePackageStatusOutdatedCheckSource.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/AuditingStoragePackageStatusOutdatedCheckSource.cs new file mode 100644 index 000000000..a72c72d9b --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/AuditingStoragePackageStatusOutdatedCheckSource.cs @@ -0,0 +1,84 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.Metadata.Catalog.Helpers; + +using CatalogStorage = NuGet.Services.Metadata.Catalog.Persistence.Storage; + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Monitoring +{ + /// + /// Fetches s from . + /// The s fetched represent packages that were deleted. + /// + /// + /// Some packages that were deleted may have been reuploaded. + /// This source does not prevent those packages from being returned. + /// + public class AuditingStoragePackageStatusOutdatedCheckSource : PackageStatusOutdatedCheckSource + { + private readonly CatalogStorage _auditingStorage; + private readonly ILogger _logger; + + private IReadOnlyCollection _cachedAuditEntries; + + public AuditingStoragePackageStatusOutdatedCheckSource( + ReadWriteCursor cursor, + CatalogStorage auditingStorage, + ILogger logger) + : base(cursor) + { + _auditingStorage = auditingStorage ?? throw new ArgumentNullException(nameof(auditingStorage)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + protected override DateTime GetCursorValue(DeletionAuditEntry package) + { + return package.TimestampUtc.Value; + } + + protected override PackageStatusOutdatedCheck GetPackageStatusOutdatedCheck(DeletionAuditEntry package) + { + return new PackageStatusOutdatedCheck(package); + } + + protected override async Task> GetPackagesToCheckAsync(DateTime since, DateTime max, int top, CancellationToken cancellationToken) + { + // Fetching audit entries is expensive, so cache them. + // When we run out of cached entries, fetch more. + if (_cachedAuditEntries == null || !_cachedAuditEntries.Any()) + { + _logger.LogInformation("Fetching audit entries from storage."); + var auditEntries = await DeletionAuditEntry + .GetAsync(_auditingStorage, CancellationToken.None, minTime: since, maxTime: max, logger: _logger); + + // A package may have multiple deleted audit entries. + // Choose only the latest for each package. + _cachedAuditEntries = auditEntries + .GroupBy(e => new FeedPackageIdentity(e.PackageId, e.PackageVersion)) + .Select(g => g.OrderByDescending(e => e.TimestampUtc).First()) + .OrderBy(e => e.TimestampUtc) + .ToList(); + + _logger.LogInformation("Cached {AuditEntryCount} audit entries.", _cachedAuditEntries.Count()); + } + else + { + _logger.LogInformation("Using cached audit entries."); + } + + var nextAuditEntries = _cachedAuditEntries.Take(top).ToList(); + _cachedAuditEntries = _cachedAuditEntries + .Skip(top) + .ToList(); + + return nextAuditEntries; + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/DatabasePackageStatusOutdatedCheckSource.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/DatabasePackageStatusOutdatedCheckSource.cs new file mode 100644 index 000000000..5750e67c8 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/DatabasePackageStatusOutdatedCheckSource.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Monitoring +{ + /// + /// Fetches from . + /// The s fetched represent the latest state for existing packages. + /// + public class DatabasePackageStatusOutdatedCheckSource : PackageStatusOutdatedCheckSource + { + private readonly IGalleryDatabaseQueryService _galleryDatabaseQueryService; + + public DatabasePackageStatusOutdatedCheckSource( + ReadWriteCursor cursor, + IGalleryDatabaseQueryService galleryDatabase) + : base(cursor) + { + _galleryDatabaseQueryService = galleryDatabase ?? throw new ArgumentNullException(nameof(galleryDatabase)); + } + + protected override DateTime GetCursorValue(FeedPackageDetails package) + { + return package.LastEditedDate; + } + + protected override PackageStatusOutdatedCheck GetPackageStatusOutdatedCheck(FeedPackageDetails package) + { + return new PackageStatusOutdatedCheck(package); + } + + /// + /// Any values greater than will be ignored by . + /// + protected override async Task> GetPackagesToCheckAsync(DateTime since, DateTime max, int top, CancellationToken cancellationToken) + { + return (await _galleryDatabaseQueryService.GetPackagesEditedSince(since, top)) + .SelectMany(p => p.Value) + .Where(p => p.LastEditedDate < max) + .ToList(); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/PackageStatusOutdatedCheck.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/PackageStatusOutdatedCheck.cs new file mode 100644 index 000000000..b08e8e4ae --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/PackageStatusOutdatedCheck.cs @@ -0,0 +1,47 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Versioning; +using System; + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Monitoring +{ + public class PackageStatusOutdatedCheck + { + public PackageStatusOutdatedCheck( + FeedPackageIdentity identity, + DateTime commitTimestamp) + { + Identity = identity; + Timestamp = commitTimestamp; + } + + public PackageStatusOutdatedCheck( + FeedPackageDetails package) + : this( + new FeedPackageIdentity( + package.PackageId, + ParseVersionString(package.PackageFullVersion)), + package.LastEditedDate) + { + } + + public PackageStatusOutdatedCheck( + DeletionAuditEntry auditEntry) + : this( + new FeedPackageIdentity(auditEntry.PackageId, ParseVersionString(auditEntry.PackageVersion)), + auditEntry.TimestampUtc.Value) + { + } + + public FeedPackageIdentity Identity { get; } + public DateTime Timestamp { get; } + + private static string ParseVersionString(string version) + { + return NuGetVersion.TryParse(version, out var parsedVersion) + ? parsedVersion.ToFullString() : version; + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/PackageStatusOutdatedCheckSource.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/PackageStatusOutdatedCheckSource.cs new file mode 100644 index 000000000..7fac82f55 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Monitoring/PackageStatusOutdatedCheckSource.cs @@ -0,0 +1,90 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Monitoring +{ + /// + /// Fetches s from a source and maintains a on it. + /// + public interface IPackageStatusOutdatedCheckSource + { + /// + /// Fetches the next batch of s. + /// + /// Up to this many s are returned. + Task> GetPackagesToCheckAsync(DateTime max, int top, CancellationToken cancellationToken); + + /// + /// Updates the based on the last batch of packages returned by . + /// If was never called, or the last batch was already processed, this method does nothing. + /// + Task MarkPackagesCheckedAsync(CancellationToken cancellationToken); + + /// + /// Moves back the to a specific time. + /// The next call to will return the batches of packages after that time. + /// If the cursor would be moving ahead, the change is not done. + /// + Task MoveBackAsync(DateTime value, CancellationToken cancellationToken); + } + + public abstract class PackageStatusOutdatedCheckSource : IPackageStatusOutdatedCheckSource + { + private readonly ReadWriteCursor _cursor; + private DateTime? _pendingCursorValue; + + public PackageStatusOutdatedCheckSource(ReadWriteCursor cursor) + { + _cursor = cursor ?? throw new ArgumentNullException(nameof(cursor)); + } + + public async Task> GetPackagesToCheckAsync(DateTime max, int top, CancellationToken cancellationToken) + { + await _cursor.LoadAsync(cancellationToken); + var packages = await GetPackagesToCheckAsync(_cursor.Value, max, top, cancellationToken); + _pendingCursorValue = packages.Any() + ? packages.Max(GetCursorValue) + : (DateTime?)null; + + return packages + .Select(GetPackageStatusOutdatedCheck) + .ToList(); + } + + public async Task MarkPackagesCheckedAsync(CancellationToken cancellationToken) + { + if (!_pendingCursorValue.HasValue) + { + return; + } + + await SetAsync(_pendingCursorValue.Value, cancellationToken); + _pendingCursorValue = null; + } + + public async Task MoveBackAsync(DateTime value, CancellationToken cancellationToken) + { + await _cursor.LoadAsync(cancellationToken); + await SetAsync(new[] { _cursor.Value, value }.Min(), cancellationToken); + } + + private Task SetAsync(DateTime value, CancellationToken cancellationToken) + { + _cursor.Value = value; + return _cursor.SaveAsync(cancellationToken); + } + + protected abstract DateTime GetCursorValue(T package); + + protected abstract PackageStatusOutdatedCheck GetPackageStatusOutdatedCheck(T package); + + protected abstract Task> GetPackagesToCheckAsync( + DateTime since, DateTime max, int top, CancellationToken cancellationToken); + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/IMonitoringNotificationService.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/IMonitoringNotificationService.cs new file mode 100644 index 000000000..b6e875d66 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/IMonitoringNotificationService.cs @@ -0,0 +1,30 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Determines behavior of the monitoring job after a package has been validated or validation failed to run on that package. + /// + public interface IMonitoringNotificationService + { + /// + /// Called whenever validation finishes on a package. + /// + /// Result of the validation. + Task OnPackageValidationFinishedAsync(PackageValidationResult result, CancellationToken token); + + /// + /// Called whenever validation failed to run on a package. + /// + /// Id of the package that could not be validated. + /// Version of the package that could not be validated. + /// Catalog entries of the package that queued the validation. + /// Exception that was thrown while running validation on the package. + Task OnPackageValidationFailedAsync(string packageId, string packageVersion, Exception e, CancellationToken token); + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/LoggerMonitoringNotificationService.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/LoggerMonitoringNotificationService.cs new file mode 100644 index 000000000..5c8ad5c4e --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Notification/LoggerMonitoringNotificationService.cs @@ -0,0 +1,60 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// that logs all validation results to an . + /// + public class LoggerMonitoringNotificationService : IMonitoringNotificationService + { + private ILogger _logger; + + public LoggerMonitoringNotificationService(ILogger logger) + { + _logger = logger; + } + + public Task OnPackageValidationFinishedAsync(PackageValidationResult result, CancellationToken token) + { + _logger.LogInformation("Finished testing {PackageId} {PackageVersion}", result.Package.Id, result.Package.Version); + var groupedResults = result.AggregateValidationResults.SelectMany(r => r.ValidationResults).GroupBy(r => r.Result); + + foreach (var resultsWithResult in groupedResults) + { + foreach (var validationResult in resultsWithResult) + { + var testResultLogString = "{PackageId} {PackageVersion}: {ValidatorName}: {TestResult}"; + var validatorName = validationResult.Validator.GetType().Name; + var testResultString = validationResult.Result.ToString(); + + switch (validationResult.Result) + { + case TestResult.Pass: + case TestResult.Skip: + _logger.LogInformation(testResultLogString, result.Package.Id, result.Package.Version, validatorName, testResultString); + break; + case TestResult.Fail: + _logger.LogError(LogEvents.ValidationFailed, validationResult.Exception, testResultLogString, result.Package.Id, result.Package.Version, validatorName, testResultString); + break; + } + } + } + + return Task.FromResult(0); + } + + public Task OnPackageValidationFailedAsync(string packageId, string packageVersion, Exception e, CancellationToken token) + { + _logger.LogError(LogEvents.ValidationFailedToRun, e, "Failed to test {PackageId} {PackageVersion}!", packageId, packageVersion); + + return Task.FromResult(0); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj new file mode 100644 index 000000000..00ab42bfb --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/NuGet.Services.Metadata.Catalog.Monitoring.csproj @@ -0,0 +1,193 @@ + + + + + + Debug + AnyCPU + {1745A383-D0BE-484B-81EB-27B20F6AC6C5} + Library + Properties + NuGet.Services.Metadata.Catalog.Monitoring + NuGet.Services.Metadata.Catalog.Monitoring + v4.7.2 + 512 + true + win + + + + .NET Foundation + https://github.com/NuGet/NuGet.Services.Metadata/blob/master/LICENSE + https://github.com/NuGet/NuGet.Services.Metadata + Monitor the package metadata catalog. + Copyright .NET Foundation + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + True + True + Strings.resx + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {e97f23b8-ecb0-4afa-b00c-015c39395fef} + NuGet.Services.Metadata.Catalog + + + + + 4.4.0 + + + 0.3.0 + runtime; build; native; contentfiles; analyzers + all + + + 2.2.0 + + + 4.8.0 + runtime; build; native; contentfiles; analyzers + all + + + 2.75.0 + + + + + ResXFileCodeGenerator + Strings.Designer.cs + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + + \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Properties/AssemblyInfo.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..b40b1e310 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Properties/AssemblyInfo.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("NuGet.Services.Metadata.Catalog.Monitoring")] +[assembly: AssemblyDescription("Monitor the package metadata catalog.")] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("1745a383-d0be-484b-81eb-27b20f6ac6c5")] + +[assembly: AssemblyCompany(".NET Foundation")] +[assembly: AssemblyProduct("NuGet Services")] +[assembly: AssemblyCopyright("\x00a9 .NET Foundation. All rights reserved.")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Providers/NonhijackedV2HttpHandlerResourceProvider.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Providers/NonhijackedV2HttpHandlerResourceProvider.cs new file mode 100644 index 000000000..e52a8f5b0 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Providers/NonhijackedV2HttpHandlerResourceProvider.cs @@ -0,0 +1,42 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using System.Threading; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class NonhijackableV2HttpHandlerResourceProvider : ResourceProvider + { + public NonhijackableV2HttpHandlerResourceProvider() : + base(typeof(HttpHandlerResource), + nameof(HttpHandlerResource), + /// Must have higher priority than + nameof(HttpHandlerResourceV3Provider)) + { + _innerResourceProvider = new HttpHandlerResourceV3Provider(); + } + + private HttpHandlerResourceV3Provider _innerResourceProvider; + + public override async Task> TryCreate(SourceRepository source, CancellationToken token) + { + var resource = await _innerResourceProvider.TryCreate(source, token); + + if (resource.Item1) + { + var clientHandler = ((HttpHandlerResource)resource.Item2).ClientHandler; + var messageHandler = new NonhijackedV2HttpMessageHandler(((HttpHandlerResource)resource.Item2).MessageHandler); + + var httpHandlerResource = new HttpHandlerResourceV3(clientHandler, messageHandler); + + resource = new Tuple(httpHandlerResource != null, httpHandlerResource); + } + + return resource; + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Providers/PackageRegistrationMetadataResourceDatabaseProvider.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Providers/PackageRegistrationMetadataResourceDatabaseProvider.cs new file mode 100644 index 000000000..99b6d9a33 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Providers/PackageRegistrationMetadataResourceDatabaseProvider.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class PackageRegistrationMetadataResourceDatabaseFeedProvider : ResourceProvider + { + private readonly IGalleryDatabaseQueryService _queryService; + + public PackageRegistrationMetadataResourceDatabaseFeedProvider( + IGalleryDatabaseQueryService queryService) : + base(typeof(IPackageRegistrationMetadataResource), + nameof(IPackageRegistrationMetadataResource)) + { + _queryService = queryService ?? throw new ArgumentNullException(nameof(queryService)); + } + + public override async Task> TryCreate(SourceRepository source, CancellationToken token) + { + IPackageRegistrationMetadataResource resource = null; + + if (await source.GetFeedType(token) == FeedType.HttpV2) + { + resource = new PackageRegistrationMetadataResourceDatabase(_queryService); + } + + return new Tuple(resource != null, resource); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Providers/PackageRegistrationMetadataResourceV3Provider.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Providers/PackageRegistrationMetadataResourceV3Provider.cs new file mode 100644 index 000000000..cbf376f02 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Providers/PackageRegistrationMetadataResourceV3Provider.cs @@ -0,0 +1,35 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class PackageRegistrationMetadataResourceV3Provider : ResourceProvider + { + public PackageRegistrationMetadataResourceV3Provider() : + base(typeof(IPackageRegistrationMetadataResource), + nameof(IPackageRegistrationMetadataResource)) + { + } + + public override async Task> TryCreate(SourceRepository source, CancellationToken token) + { + IPackageRegistrationMetadataResource resource = null; + + if (await source.GetFeedType(token) == FeedType.HttpV3) + { + var registration = await source.GetResourceAsync(token); + var httpSourceResource = await source.GetResourceAsync(token); + + resource = new PackageRegistrationMetadataResourceV3(registration, httpSourceResource.HttpSource); + } + + return new Tuple(resource != null, resource); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Providers/PackageTimestampMetadataResourceDatabaseProvider.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Providers/PackageTimestampMetadataResourceDatabaseProvider.cs new file mode 100644 index 000000000..961d5f08e --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Providers/PackageTimestampMetadataResourceDatabaseProvider.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class PackageTimestampMetadataResourceDatabaseProvider : ResourceProvider + { + private readonly IGalleryDatabaseQueryService _galleryDatabase; + + public PackageTimestampMetadataResourceDatabaseProvider( + IGalleryDatabaseQueryService galleryDatabaseQueryService, + ILoggerFactory loggerFactory) + : base(typeof(IPackageTimestampMetadataResource)) + { + _galleryDatabase = galleryDatabaseQueryService ?? throw new ArgumentNullException(nameof(galleryDatabaseQueryService)); + _loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory)); + } + + private readonly ILoggerFactory _loggerFactory; + + public override async Task> TryCreate(SourceRepository source, CancellationToken token) + { + PackageTimestampMetadataResourceDatabase resource = null; + + if (await source.GetFeedType(token) == FeedType.HttpV2) + { + resource = new PackageTimestampMetadataResourceDatabase( + _galleryDatabase, + _loggerFactory.CreateLogger()); + } + + return new Tuple(resource != null, resource); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/IPackageRegistrationMetadataResource.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/IPackageRegistrationMetadataResource.cs new file mode 100644 index 000000000..bbefce392 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/IPackageRegistrationMetadataResource.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading; +using System.Threading.Tasks; +using NuGet.Common; +using NuGet.Packaging.Core; +using NuGet.Protocol.Core.Types; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Aggregates information about index and leaf metadata. + /// + public interface IPackageRegistrationMetadataResource : INuGetResource + { + Task GetIndexAsync(PackageIdentity package, ILogger log, CancellationToken token); + + Task GetLeafAsync(PackageIdentity package, ILogger log, CancellationToken token); + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/IPackageTimestampMetadataResource.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/IPackageTimestampMetadataResource.cs new file mode 100644 index 000000000..8688064a9 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/IPackageTimestampMetadataResource.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using NuGet.Protocol.Core.Types; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public interface IPackageTimestampMetadataResource : INuGetResource + { + Task GetAsync(ValidationContext context); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/PackageRegistrationMetadataResourceDatabaseFeed.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/PackageRegistrationMetadataResourceDatabaseFeed.cs new file mode 100644 index 000000000..98e20733c --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/PackageRegistrationMetadataResourceDatabaseFeed.cs @@ -0,0 +1,60 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Common; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class PackageRegistrationMetadataResourceDatabase : IPackageRegistrationMetadataResource + { + private readonly IGalleryDatabaseQueryService _galleryDatabase; + + public PackageRegistrationMetadataResourceDatabase( + IGalleryDatabaseQueryService galleryDatabase) + { + _galleryDatabase = galleryDatabase ?? throw new ArgumentNullException(nameof(galleryDatabase)); + } + + /// + /// Returns a that represents how a package appears in the database. + /// + public async Task GetIndexAsync(PackageIdentity package, ILogger log, CancellationToken token) + { + try + { + var feedPackage = await GetPackageAsync(package); + return feedPackage != null ? new PackageRegistrationIndexMetadata(feedPackage) : null; + } + catch (Exception e) + { + throw new ValidationException($"Could not fetch {nameof(PackageRegistrationIndexMetadata)} from database!", e); + } + } + + /// + /// Returns a that represents how a package appears in the database. + /// + public async Task GetLeafAsync(PackageIdentity package, ILogger log, CancellationToken token) + { + try + { + var feedPackage = await GetPackageAsync(package); + return feedPackage != null ? new PackageRegistrationLeafMetadata(feedPackage) : null; + } + catch (Exception e) + { + throw new ValidationException($"Could not fetch {nameof(PackageRegistrationLeafMetadata)} from database!", e); + } + } + + private Task GetPackageAsync(PackageIdentity package) + { + return _galleryDatabase.GetPackageOrNull(package.Id, package.Version.ToNormalizedString()); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/PackageRegistrationMetadataResourceV3.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/PackageRegistrationMetadataResourceV3.cs new file mode 100644 index 000000000..bdb51618b --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/PackageRegistrationMetadataResourceV3.cs @@ -0,0 +1,75 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NuGet.Common; +using NuGet.Packaging.Core; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class PackageRegistrationMetadataResourceV3 : IPackageRegistrationMetadataResource + { + private RegistrationResourceV3 _registration; + private HttpSource _client; + + public PackageRegistrationMetadataResourceV3( + RegistrationResourceV3 registration, + HttpSource client) + { + _registration = registration; + _client = client; + } + + public async Task GetIndexAsync(PackageIdentity package, ILogger log, CancellationToken token) + { + try + { + var feedPackage = await GetPackageFromIndexAsync(package, log, token); + return feedPackage != null ? + JsonConvert.DeserializeObject(feedPackage.ToString()) : + null; + } + catch (Exception e) + { + throw new ValidationException($"Could not fetch {nameof(PackageRegistrationIndexMetadata)} from V3!", e); + } + } + + public async Task GetLeafAsync(PackageIdentity package, ILogger log, CancellationToken token) + { + try + { + var feedPackage = await GetPackageFromLeafAsync(package, log, token); + return feedPackage != null ? + JsonConvert.DeserializeObject(feedPackage.ToString()) : + null; + } + catch (Exception e) + { + throw new ValidationException($"Could not fetch {nameof(PackageRegistrationLeafMetadata)} from V3!", e); + } + } + + private Task GetPackageFromIndexAsync(PackageIdentity package, ILogger log, CancellationToken token) + { + // If the registration index is missing, this will return null. + return _registration.GetPackageMetadata(package, NullSourceCacheContext.Instance, log, token); + } + + private async Task GetPackageFromLeafAsync(PackageIdentity package, ILogger log, CancellationToken token) + { + /// If the registration leaf is missing, will cause this to return null. + return await _client.GetJObjectAsync( + new HttpSourceRequest( + _registration.GetUri(package), log) + { IgnoreNotFounds = true }, + log, token); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/PackageTimestampMetadataResourceDatabase.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/PackageTimestampMetadataResourceDatabase.cs new file mode 100644 index 000000000..c90021d24 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Resources/PackageTimestampMetadataResourceDatabase.cs @@ -0,0 +1,51 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class PackageTimestampMetadataResourceDatabase : IPackageTimestampMetadataResource + { + private readonly IGalleryDatabaseQueryService _galleryDatabase; + private readonly ILogger _logger; + + public PackageTimestampMetadataResourceDatabase( + IGalleryDatabaseQueryService galleryDatabase, + ILogger logger) + { + _galleryDatabase = galleryDatabase ?? throw new ArgumentNullException(nameof(galleryDatabase)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + /// Queries the gallery database for the package specified by the and returns a . + /// If the package is missing from the repository, returns the package's deletion audit record timestamp. + /// + public async Task GetAsync(ValidationContext context) + { + var feedPackageDetails = await _galleryDatabase.GetPackageOrNull( + context.Package.Id, + context.Package.Version.ToNormalizedString()); + + if (feedPackageDetails != null) + { + return PackageTimestampMetadata.CreateForExistingPackage( + feedPackageDetails.CreatedDate, + feedPackageDetails.LastEditedDate); + } + + DateTime? deleted = null; + if (context.DeletionAuditEntries.Any()) + { + deleted = context.DeletionAuditEntries.Max(entry => entry.TimestampUtc); + } + + return PackageTimestampMetadata.CreateForMissingPackage(deleted); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/IPackageMonitoringStatusService.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/IPackageMonitoringStatusService.cs new file mode 100644 index 000000000..caf740d51 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/IPackageMonitoringStatusService.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Used to manage the status of packages that validation has ran against. + /// + public interface IPackageMonitoringStatusService + { + /// + /// Returns a list of every package that has been monitored and its . + /// + Task> ListAsync(CancellationToken token); + + /// + /// Returns the validation status of a package. + /// If validation has not yet been run on the package, returns null. + /// + Task GetAsync(FeedPackageIdentity package, CancellationToken token); + + /// + /// Returns the status of all packages that have a specified . + /// + Task> GetAsync(PackageState type, CancellationToken token); + + /// + /// Updates the status of a package. + /// + Task UpdateAsync(PackageMonitoringStatus status, CancellationToken token); + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatus.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatus.cs new file mode 100644 index 000000000..5aa4fcfcb --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatus.cs @@ -0,0 +1,107 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.WindowsAzure.Storage; +using Newtonsoft.Json; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// The validation status of a package. Contains whether or not validation ran, and if the validation succeeded. + /// + public class PackageMonitoringStatus + { + /// + /// The identity of the package. + /// + [JsonProperty("package")] + public FeedPackageIdentity Package { get; } + + [JsonProperty("state")] + public PackageState State + { + get + { + if (ValidationException != null || HasResultsOfType(TestResult.Fail)) + { + return PackageState.Invalid; + } + + if (HasResultsOfType(TestResult.Pending)) + { + return PackageState.Unknown; + } + + return PackageState.Valid; + } + } + + /// + /// If validation ran, the results of the validation. + /// + [JsonProperty("validationResult")] + public PackageValidationResult ValidationResult { get; } + + /// + /// If validation failed to run, the exception that was thrown. + /// + [JsonProperty("validationException")] + public Exception ValidationException { get; } + + /// + /// If this status was loaded from storage, this access condition should be used to overwrite it. + /// + [JsonIgnore] + public AccessCondition AccessCondition { get; set; } + + /// + /// When this status is saved to storage, the save operation should use these s to save to the containers associated with each . + /// + [JsonIgnore] + public IDictionary ExistingState { get; } + + [JsonConstructor] + public PackageMonitoringStatus(FeedPackageIdentity package, PackageValidationResult validationResult, Exception validationException) + : this() + { + Package = package; + ValidationResult = validationResult; + ValidationException = validationException; + } + + public PackageMonitoringStatus(PackageValidationResult result) + : this() + { + ValidationResult = result ?? throw new ArgumentNullException(nameof(result)); + Package = new FeedPackageIdentity(result.Package); + } + + public PackageMonitoringStatus(FeedPackageIdentity package, Exception exception) + : this() + { + Package = package ?? throw new ArgumentNullException(nameof(package)); + ValidationException = exception ?? throw new ArgumentNullException(nameof(exception)); + } + + private PackageMonitoringStatus() + { + AccessCondition = PackageMonitoringStatusAccessConditionHelper.FromUnknown(); + ExistingState = new Dictionary(); + foreach (var state in Enum.GetValues(typeof(PackageState)).Cast()) + { + ExistingState[state] = PackageMonitoringStatusAccessConditionHelper.FromUnknown(); + } + } + + private bool HasResultsOfType(TestResult result) + { + return ValidationResult.AggregateValidationResults.Any( + r => r.ValidationResults.Any( + v => v.Result == result)); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusAccessConditionHelper.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusAccessConditionHelper.cs new file mode 100644 index 000000000..1e555009b --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusAccessConditionHelper.cs @@ -0,0 +1,44 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.WindowsAzure.Storage; +using NuGet.Services.Metadata.Catalog.Persistence; +using System; +using System.Linq; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public static class PackageMonitoringStatusAccessConditionHelper + { + public static AccessCondition FromContent(StorageContent content) + { + var eTag = (content as StringStorageContentWithETag)?.ETag; + if (eTag == null) + { + return AccessCondition.GenerateEmptyCondition(); + } + else + { + return AccessCondition.GenerateIfMatchCondition(eTag); + } + } + + public static void UpdateFromExisting(PackageMonitoringStatus status, PackageMonitoringStatus existingStatus) + { + foreach (var state in Enum.GetValues(typeof(PackageState)).Cast()) + { + status.ExistingState[state] = AccessCondition.GenerateIfNotExistsCondition(); + } + + if (existingStatus != null) + { + status.ExistingState[existingStatus.State] = existingStatus.AccessCondition; + } + } + + public static AccessCondition FromUnknown() + { + return AccessCondition.GenerateEmptyCondition(); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusListItem.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusListItem.cs new file mode 100644 index 000000000..b85a9edc2 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusListItem.cs @@ -0,0 +1,24 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Part of the metadata of a as returned by . + /// The full can be returned by calling . + /// + public class PackageMonitoringStatusListItem + { + public FeedPackageIdentity Package { get; } + + public PackageState State { get; } + + public PackageMonitoringStatusListItem(FeedPackageIdentity package, PackageState state) + { + Package = package; + State = state; + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusService.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusService.cs new file mode 100644 index 000000000..43ce06a82 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageMonitoringStatusService.cs @@ -0,0 +1,284 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.WindowsAzure.Storage; +using Newtonsoft.Json; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; + +using CatalogStorage = NuGet.Services.Metadata.Catalog.Persistence.Storage; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Manages the storage and access of the status of packages that validation has run against. + /// + public class PackageMonitoringStatusService : IPackageMonitoringStatusService + { + private static readonly string[] _packageStateNames = Enum.GetNames(typeof(PackageState)); + private static readonly Array _packageStateValues = Enum.GetValues(typeof(PackageState)); + + private readonly ILogger _logger; + + /// + /// The to use to save status of packages. + /// + private readonly IStorageFactory _storageFactory; + + public PackageMonitoringStatusService(IStorageFactory storageFactory, ILogger logger) + { + _logger = logger; + _storageFactory = storageFactory; + } + + public async Task> ListAsync(CancellationToken token) + { + var listTasks = + _packageStateNames + .Select(state => GetListItems(state, token)) + .ToList(); + + return + (await Task.WhenAll(listTasks)) + .Where(list => list != null && list.Any()) + .Aggregate((current, next) => current.Concat(next)) + .Where(i => i != null); + } + + private async Task> GetListItems(string state, CancellationToken token) + { + var storage = GetStorage(state); + var list = await storage.ListAsync(token); + return list.Select(item => + { + try + { + return new PackageMonitoringStatusListItem(ParsePackageUri(item.Uri), (PackageState)Enum.Parse(typeof(PackageState), state)); + } + catch (Exception e) + { + _logger.LogWarning("Failed to parse list item {ItemUri}: {Exception}", item.Uri, e); + return null; + } + }); + } + + public async Task GetAsync(FeedPackageIdentity package, CancellationToken token) + { + var statusTasks = + _packageStateNames + .Select(state => GetPackageAsync(GetStorage(state), package, token)) + .ToList(); + + var statuses = + (await Task.WhenAll(statusTasks)) + .Where(s => s != null); + + if (!statuses.Any()) + { + return null; + } + + // If more than one status exist for a single package, find the status with the latest timestamp. + // We then must delete the other statuses, so that we can later update the storage safely. + var statusesWithTimeStamps = statuses.Where(s => s.ValidationResult != null && s.ValidationResult.CatalogEntries != null && s.ValidationResult.CatalogEntries.Any()); + IEnumerable orderedStatuses; + if (statusesWithTimeStamps.Any()) + { + orderedStatuses = statusesWithTimeStamps.OrderByDescending(s => s.ValidationResult.CatalogEntries.Max(c => c.CommitTimeStamp)); + } + else + { + // No statuses have timestamps (they all failed to process). + // Because they are all in a bad state, choose an arbitrary one. + orderedStatuses = statuses; + } + + var result = orderedStatuses.First(); + foreach (var statusToDelete in orderedStatuses.Skip(1)) + { + await DeleteAsync(statusToDelete.Package, statusToDelete.State, statusToDelete.AccessCondition, token); + } + + return result; + } + + public async Task> GetAsync(PackageState state, CancellationToken token) + { + var packageStatuses = new List(); + + var storage = GetStorage(state); + + var statusTasks = + (await storage.ListAsync(token)) + .Select(listItem => GetPackageAsync(storage, listItem.Uri, token)) + .ToList(); + + return + (await Task.WhenAll(statusTasks)) + .Where(s => s != null); + } + + public async Task UpdateAsync(PackageMonitoringStatus status, CancellationToken token) + { + // Save the new status first. + // If we save it after deleting the existing status, the save could fail and then we'd lose the data. + await SaveAsync(status, token); + + // Delete the other statuses. + foreach (int stateInt in _packageStateValues) + { + var state = (PackageState)stateInt; + if (state != status.State) + { + // Delete the existing status. + await DeleteAsync( + status.Package, + state, + status.ExistingState[state], + token); + } + } + } + + private async Task SaveAsync(PackageMonitoringStatus status, CancellationToken token) + { + var storage = GetStorage(status.State); + + var packageStatusJson = JsonConvert.SerializeObject(status, JsonSerializerUtility.SerializerSettings); + var storageContent = new StringStorageContentWithAccessCondition( + packageStatusJson, + status.ExistingState[status.State], + "application/json"); + + var packageUri = GetPackageUri(storage, status.Package); + + await storage.SaveAsync(packageUri, storageContent, token); + } + + private async Task DeleteAsync(FeedPackageIdentity package, PackageState state, AccessCondition accessCondition, CancellationToken token) + { + var storage = GetStorage(state); + if (!storage.Exists(GetPackageFileName(package))) + { + return; + } + + await storage.DeleteAsync( + GetPackageUri(storage, package), + token, + new DeleteRequestOptionsWithAccessCondition( + accessCondition)); + } + + private CatalogStorage GetStorage(PackageState state) + { + return GetStorage(state.ToString()); + } + + private CatalogStorage GetStorage(string stateString) + { + return _storageFactory.Create(stateString.ToLowerInvariant()); + } + + private Uri GetPackageUri(CatalogStorage storage, FeedPackageIdentity package) + { + return storage.ResolveUri(GetPackageFileName(package)); + } + + private string GetPackageFileName(FeedPackageIdentity package) + { + var idString = package.Id.ToLowerInvariant(); + var versionString = package.Version.ToLowerInvariant(); + + return $"{idString}/" + + $"{idString}.{versionString}.json"; + } + + /// + /// Parses a into a . + /// + /// The must end with "/{id}/{id}.{version}.json" + /// + private FeedPackageIdentity ParsePackageUri(Uri packageUri) + { + var uriSegments = packageUri.Segments; + // The second to last segment is the id. + var id = uriSegments[uriSegments.Length - 2].Trim('/'); + + // The last segment is {id}.{version}.json. + // Remove the id and the "." from the beginning. + var version = uriSegments[uriSegments.Length - 1].Substring(id.Length + ".".Length); + // Remove the ".json" from the end. + version = version.Substring(0, version.Length - ".json".Length); + + return new FeedPackageIdentity(id, version); + } + + private Task GetPackageAsync(CatalogStorage storage, FeedPackageIdentity package, CancellationToken token) + { + return GetPackageAsync(storage, GetPackageFileName(package), token); + } + + private Task GetPackageAsync(CatalogStorage storage, string fileName, CancellationToken token) + { + if (!storage.Exists(fileName)) + { + return Task.FromResult(null); + } + + return GetPackageAsync(storage, storage.ResolveUri(fileName), token); + } + + private async Task GetPackageAsync(CatalogStorage storage, Uri packageUri, CancellationToken token) + { + try + { + var content = await storage.LoadAsync(packageUri, token); + string statusString = null; + using (var stream = content.GetContentStream()) + { + using (var reader = new StreamReader(stream)) + { + statusString = await reader.ReadToEndAsync(); + } + } + + var status = JsonConvert.DeserializeObject(statusString, JsonSerializerUtility.SerializerSettings); + status.AccessCondition = PackageMonitoringStatusAccessConditionHelper.FromContent(content); + return status; + } + catch (Exception deserializationException) + { + _logger.LogWarning( + LogEvents.StatusDeserializationFailure, + deserializationException, + "Unable to deserialize package status from {PackageUri}!", + packageUri); + + try + { + /// Construct a from the with this as the exception. + return new PackageMonitoringStatus(ParsePackageUri(packageUri), new StatusDeserializationException(deserializationException)); + } + catch (Exception uriParsingException) + { + _logger.LogError( + LogEvents.StatusDeserializationFatalFailure, + new AggregateException(deserializationException, uriParsingException), + "Unable to get package id and version from {PackageUri}!", + packageUri); + + return null; + } + } + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageState.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageState.cs new file mode 100644 index 000000000..92f1af841 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/PackageState.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// The state of a package's metadata. + /// + public enum PackageState + { + /// + /// The package is in a valid state. + /// Its metadata appears exactly as expected. + /// + Valid, + + /// + /// The package is in an invalid state. + /// Its metadata is missing or in an unexpected state. + /// + Invalid, + + /// + /// The package's state could not be determined. + /// Its metadata is in an unknown state. + /// + /// + /// Typically, a package is in this state if there is a newer catalog entry for a package that needs to be processed. + /// If a package is in this state for an extended period of time, there is likely a problem with it or the monitoring pipeline. + /// + Unknown, + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/StatusDeserializationException.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/StatusDeserializationException.cs new file mode 100644 index 000000000..d1fcc6721 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Status/StatusDeserializationException.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + [Serializable] + public class StatusDeserializationException : Exception + { + public StatusDeserializationException(Exception e) + : base("Failed to deserialize the status!", e) + { + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Strings.Designer.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Strings.Designer.cs new file mode 100644 index 000000000..0d83de7d8 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Strings.Designer.cs @@ -0,0 +1,72 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace NuGet.Services.Metadata.Catalog.Monitoring { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "15.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + internal class Strings { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Strings() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("NuGet.Services.Metadata.Catalog.Monitoring.Strings", typeof(Strings).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to The argument must not be null or empty.. + /// + internal static string ArgumentMustNotBeNullOrEmpty { + get { + return ResourceManager.GetString("ArgumentMustNotBeNullOrEmpty", resourceCulture); + } + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Strings.resx b/src/NuGet.Services.Metadata.Catalog.Monitoring/Strings.resx new file mode 100644 index 000000000..04429ed2b --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Strings.resx @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + The argument must not be null or empty. + + \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/CommonLogger.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/CommonLogger.cs new file mode 100644 index 000000000..4ac1139ab --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/CommonLogger.cs @@ -0,0 +1,115 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Common; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// wrapper for . + /// + public class CommonLogger : Common.ILogger + { + // This event ID is believed to be unused anywhere else but is otherwise arbitrary. + private const int DefaultLogEventId = 23847; + private static EventId DefaultClientLogEvent = new EventId(DefaultLogEventId); + + public CommonLogger(Microsoft.Extensions.Logging.ILogger logger) + { + InternalLogger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public Microsoft.Extensions.Logging.ILogger InternalLogger { get; private set; } + + public void LogDebug(string data) + { + InternalLogger.LogDebug(data); + } + + public void LogVerbose(string data) + { + InternalLogger.LogInformation(data); + } + + public void LogInformation(string data) + { + InternalLogger.LogInformation(data); + } + + public void LogMinimal(string data) + { + InternalLogger.LogInformation(data); + } + + public void LogWarning(string data) + { + InternalLogger.LogWarning(data); + } + + public void LogError(string data) + { + InternalLogger.LogError(data); + } + + public void LogInformationSummary(string data) + { + InternalLogger.LogInformation(data); + } + + public void LogErrorSummary(string data) + { + InternalLogger.LogError(data); + } + + public void Log(Common.LogLevel level, string data) + { + InternalLogger.Log(GetLogLevel(level), DefaultClientLogEvent, data, null, (str, ex) => str); + } + + public Task LogAsync(Common.LogLevel level, string data) + { + InternalLogger.Log(GetLogLevel(level), DefaultClientLogEvent, data, null, (str, ex) => str); + return Task.FromResult(null); + } + + public void Log(ILogMessage message) + { + InternalLogger.Log(GetLogLevel(message.Level), new EventId((int)message.Code), message.Message, null, (str, ex) => str); + } + + public Task LogAsync(ILogMessage message) + { + InternalLogger.Log(GetLogLevel(message.Level), new EventId((int)message.Code), message.Message, null, (str, ex) => str); + return Task.FromResult(null); + } + + private static Microsoft.Extensions.Logging.LogLevel GetLogLevel(Common.LogLevel logLevel) + { + switch (logLevel) + { + case Common.LogLevel.Debug: + return Microsoft.Extensions.Logging.LogLevel.Debug; + + case Common.LogLevel.Verbose: + return Microsoft.Extensions.Logging.LogLevel.Information; + + case Common.LogLevel.Information: + return Microsoft.Extensions.Logging.LogLevel.Information; + + case Common.LogLevel.Minimal: + return Microsoft.Extensions.Logging.LogLevel.Information; + + case Common.LogLevel.Warning: + return Microsoft.Extensions.Logging.LogLevel.Warning; + + case Common.LogLevel.Error: + return Microsoft.Extensions.Logging.LogLevel.Error; + } + + return Microsoft.Extensions.Logging.LogLevel.None; + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/ContainerBuilderExtensions.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/ContainerBuilderExtensions.cs new file mode 100644 index 000000000..cc05144b3 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/ContainerBuilderExtensions.cs @@ -0,0 +1,244 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net.Http; +using Autofac; +using Microsoft.Extensions.Logging; +using NuGet.Configuration; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring.Validation.Test.Registration; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public static class ContainerBuilderExtensions + { + public static void RegisterValidatorConfiguration( + this ContainerBuilder builder, + ValidatorConfiguration validatorConfig) + { + if (validatorConfig == null) + { + throw new ArgumentNullException(nameof(validatorConfig)); + } + + builder + .RegisterInstance(validatorConfig) + .AsSelf(); + } + + public static void RegisterEndpointConfiguration( + this ContainerBuilder builder, + EndpointConfiguration endpointConfig) + { + if (endpointConfig == null) + { + throw new ArgumentNullException(nameof(endpointConfig)); + } + + builder + .RegisterInstance(endpointConfig) + .AsSelf(); + } + + public static void RegisterMessageHandlerFactory( + this ContainerBuilder builder, + Func messageHandlerFactory) + { + if (messageHandlerFactory == null) + { + throw new ArgumentNullException(nameof(messageHandlerFactory)); + } + + builder + .RegisterInstance(messageHandlerFactory) + .As>(); + } + + public static void RegisterEndpoints( + this ContainerBuilder builder, + EndpointConfiguration endpointConfig) + { + builder.RegisterEndpoint(); + builder.RegisterEndpoint(); + builder.RegisterEndpoint(); + builder.RegisterSearchEndpoints(endpointConfig); + } + + private static void RegisterSearchEndpoints(this ContainerBuilder builder, EndpointConfiguration endpointConfig) + { + foreach (var pair in endpointConfig.InstanceNameToSearchConfiguration) + { + var config = pair.Value; + + builder + .Register(c => new SearchEndpoint( + pair.Key, + pair.Value.CursorUris, + pair.Value.BaseUri, + c.Resolve>())) + .As() + .Keyed(pair.Key); + + builder + .Register(c => new EndpointValidator( + c.ResolveKeyed(pair.Key), + c.ResolveKeyed>>(pair.Key), + c.Resolve>())) + .As() + .Keyed>(pair.Key); + } + } + + private static void RegisterEndpoint(this ContainerBuilder builder) + where T : class, IEndpoint + { + builder + .RegisterType() + .AsSelf() + .As(); + + builder + .RegisterType>() + .AsSelf() + .As(); + } + + public static void RegisterValidators(this ContainerBuilder builder, EndpointConfiguration endpointConfig) + { + // Catalog validators + builder.RegisterValidator(); + + // Registration validators + builder.RegisterValidator(); + builder.RegisterValidator(); + builder.RegisterValidator(); + builder.RegisterValidator(); + builder.RegisterValidator(); + builder.RegisterValidator(); + + // Flat-container validators + builder.RegisterValidator(); + + // Search validators + foreach (var pair in endpointConfig.InstanceNameToSearchConfiguration) + { + builder.RegisterSearchValidator(pair.Key); + } + } + + private static void RegisterSearchValidator(this ContainerBuilder builder, string instanceName) + where TValidator : IValidator + { + builder + .RegisterType() + .Keyed>(instanceName) + .WithParameter( + (pi, ctx) => pi.ParameterType == typeof(SearchEndpoint), + (pi, ctx) => ctx.ResolveKeyed(instanceName)); + } + + private static void RegisterValidator(this ContainerBuilder builder) + where TEndpoint : IEndpoint + where TValidator : IValidator + { + builder + .RegisterType() + .As>(); + } + + public static void RegisterSourceRepositories( + this ContainerBuilder builder, + string galleryUrl, + string indexUrl, + IGalleryDatabaseQueryService galleryDatabase) + { + if (string.IsNullOrEmpty(galleryUrl)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(galleryUrl)); + } + + if (string.IsNullOrEmpty(indexUrl)) + { + throw new ArgumentException(Strings.ArgumentMustNotBeNullOrEmpty, nameof(indexUrl)); + } + + builder + .RegisterInstance(new PackageSource(galleryUrl)) + .Keyed(FeedType.HttpV2); + + builder + .RegisterInstance(new PackageSource(indexUrl)) + .Keyed(FeedType.HttpV3); + + builder + .RegisterInstance(galleryDatabase) + .AsImplementedInterfaces() + .AsSelf(); + + builder.RegisterDefaultResourceProviders(); + builder.RegisterV2ResourceProviders(); + builder.RegisterV3ResourceProviders(); + + builder.RegisterSourceRepository(FeedType.HttpV2); + builder.RegisterSourceRepository(FeedType.HttpV3); + + builder + .RegisterType() + .WithParameter( + (pi, ctx) => pi.Name == "v2", + (pi, ctx) => ctx.ResolveKeyed(FeedType.HttpV2)) + .WithParameter( + (pi, ctx) => pi.Name == "v3", + (pi, ctx) => ctx.ResolveKeyed(FeedType.HttpV3)) + .AsSelf(); + } + + private static void RegisterSourceRepository(this ContainerBuilder builder, FeedType type) + { + builder + .RegisterType() + .WithParameter( + (pi, ctx) => pi.ParameterType == typeof(PackageSource), + (pi, ctx) => ctx.ResolveKeyed(type)) + .WithParameter( + (pi, ctx) => pi.ParameterType == typeof(IEnumerable), + (pi, ctx) => ctx.ResolveKeyed>(type)) + .WithParameter(TypedParameter.From(type)) + .Keyed(type); + } + + private static void RegisterDefaultResourceProviders(this ContainerBuilder builder) + { + foreach (var provider in Repository.Provider.GetCoreV3()) + { + builder + .RegisterInstance(provider) + .As>(); + } + } + + private static void RegisterV2ResourceProviders(this ContainerBuilder builder) + { + builder.RegisterResourceProvider(FeedType.HttpV2); + builder.RegisterResourceProvider(FeedType.HttpV2); + builder.RegisterResourceProvider(FeedType.HttpV2); + } + + private static void RegisterV3ResourceProviders(this ContainerBuilder builder) + { + builder.RegisterResourceProvider(FeedType.HttpV3); + } + + private static void RegisterResourceProvider(this ContainerBuilder builder, FeedType type) + where TProvider : INuGetResourceProvider + { + builder + .RegisterType() + .Keyed(type); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/ILoggerExtensions.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/ILoggerExtensions.cs new file mode 100644 index 000000000..2160e5337 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/ILoggerExtensions.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public static class ILoggerExtensions + { + /// + /// Wraps as a . + /// + public static Common.ILogger AsCommon(this ILogger logger) + { + return new CommonLogger(logger); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/ILoggerFactoryExtensions.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/ILoggerFactoryExtensions.cs new file mode 100644 index 000000000..d93be1a4f --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/ILoggerFactoryExtensions.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public static class ILoggerFactoryExtensions + { + /// + /// Replacement for that creates an . + /// + public static ILogger CreateTypedLogger(this ILoggerFactory loggerFactory, Type type) + { + var typedCreateLoggerMethod = + typeof(LoggerFactoryExtensions) + .GetMethods() + .SingleOrDefault(m => + m.Name == nameof(LoggerFactoryExtensions.CreateLogger) && + m.IsGenericMethod); + + return typedCreateLoggerMethod + .MakeGenericMethod(type) + .Invoke(null, new object[] { loggerFactory }) as ILogger; + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/JsonSerializerUtility.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/JsonSerializerUtility.cs new file mode 100644 index 000000000..90a30b0d0 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/JsonSerializerUtility.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using NuGet.Protocol; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public static class JsonSerializerUtility + { + /// + /// The to use. + /// + public static JsonSerializerSettings SerializerSettings + { + get + { + var settings = new JsonSerializerSettings(); + + settings.Converters.Add(new NuGetVersionConverter()); + settings.Converters.Add(new StringEnumConverter()); + + return settings; + } + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/LogEvents.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/LogEvents.cs new file mode 100644 index 000000000..1c7e9dd86 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/LogEvents.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public static class LogEvents + { + public static EventId ValidationFailed = new EventId(900, "Validation failed!"); + public static EventId ValidationFailedToRun = new EventId(901, "Failed to run validation!"); + public static EventId ValidationFailedToInitialize = new EventId(902, "Failed to initialize validation!"); + + public static EventId StatusDeserializationFailure = new EventId(903, "Status deserialization failed!"); + public static EventId StatusDeserializationFatalFailure = new EventId(904, "Status deserialization failed, and was unable to parse id and version from filename!"); + + public static EventId QueueMessageFatalFailure = new EventId(905, "Failed to process queue message"); + public static EventId QueueMessageRemovalFailure = new EventId(906, "Failed to remove queue message"); + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/NullableNuGetVersionConverter.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/NullableNuGetVersionConverter.cs new file mode 100644 index 000000000..9fa877513 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/NullableNuGetVersionConverter.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NuGet.Protocol; +using NuGet.Versioning; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// that accepts null s. + /// Calls into existing when non-null. + /// + public class NullableNuGetVersionConverter : NuGetVersionConverter + { + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.Null || reader.Value == null) + { + return null; + } + + if (reader.TokenType == JsonToken.String && + string.IsNullOrEmpty(new JValue(reader.Value).ToString())) + { + return null; + } + + return base.ReadJson(reader, objectType, existingValue, serializer); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/SafeExceptionConverter.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/SafeExceptionConverter.cs new file mode 100644 index 000000000..f77b5c35b --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Utility/SafeExceptionConverter.cs @@ -0,0 +1,64 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Runtime.Serialization; +using Newtonsoft.Json; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// for converting exceptions safely. + /// If the exception fails to deserialize, returns an instead of failing. + /// + public class SafeExceptionConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return typeof(Exception).IsAssignableFrom(objectType); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + try + { + return serializer.Deserialize(reader, objectType); + } + catch (Exception e) + { + // When deserializing the exception fails, we don't want to fail deserialization of the entire object. + // Return the exception that was thrown instead. + return e; + } + } + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + var ex = value as Exception; + var serializableEx = new Wrapper(ex); + serializer.Serialize(writer, serializableEx); + } + + /// + /// This class needs to exist because System.Exception is not marked with as Serializable attribute + /// and old JSON.NET behaviour was incorrectly treating all ISerializable as Serializable + /// This was changed between 10.x and 11.x. See https://github.com/JamesNK/Newtonsoft.Json/issues/1622 for more details + /// For our purposes, passing through the GetObjectData call from a custom class marked as serialable is sufficient + /// + [Serializable] + private class Wrapper : ISerializable + { + private Exception _internalException; + + public Wrapper(Exception ex) + { + _internalException = ex; + } + + public void GetObjectData(SerializationInfo info, StreamingContext context) + { + _internalException.GetObjectData(info, context); + } + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/AggregateValidationResult.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/AggregateValidationResult.cs new file mode 100644 index 000000000..b581193ba --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/AggregateValidationResult.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class AggregateValidationResult + { + [JsonProperty("validator")] + public IValidatorIdentity AggregateValidator { get; } + + [JsonProperty("results")] + public IEnumerable ValidationResults { get; } + + public AggregateValidationResult( + IValidatorIdentity aggregateValidator, + IEnumerable validationResults) + { + AggregateValidator = aggregateValidator; + ValidationResults = validationResults; + } + + [JsonConstructor] + public AggregateValidationResult( + ValidatorIdentity validator, + IEnumerable results) + { + AggregateValidator = validator; + ValidationResults = results; + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/PackageValidationResult.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/PackageValidationResult.cs new file mode 100644 index 000000000..e7d662f8a --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/PackageValidationResult.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Newtonsoft.Json; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class PackageValidationResult + { + [JsonProperty("package")] + public PackageIdentity Package { get; } + + [JsonProperty("catalogEntries")] + public IEnumerable CatalogEntries { get; } + + [JsonProperty("deletionAuditEntries")] + public IEnumerable DeletionAuditEntries { get; } + + [JsonProperty("results")] + public IEnumerable AggregateValidationResults { get; } + + public PackageValidationResult(ValidationContext context, IEnumerable results) + : this(context.Package, context.Entries, context.DeletionAuditEntries, results) + { + } + + [JsonConstructor] + public PackageValidationResult( + PackageIdentity package, + IEnumerable catalogEntries, + IEnumerable deletionAuditEntries, + IEnumerable results) + { + Package = package; + CatalogEntries = catalogEntries; + DeletionAuditEntries = deletionAuditEntries; + AggregateValidationResults = results; + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/ShouldRunTestResult.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/ShouldRunTestResult.cs new file mode 100644 index 000000000..f77ec4393 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/ShouldRunTestResult.cs @@ -0,0 +1,49 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// The possible outcomes of . + /// + public enum ShouldRunTestResult + { + /// + /// The test should run. + /// + Yes, + + /// + /// The test can be safely skipped. + /// + No, + + /// + /// The test must be attempted again when more information is available. + /// + /// + /// Typically, this suggests that there is a newer catalog entry for this package. When that catalog entry is processed, a different result should be returned. + /// + RetryLater, + } + + public static class ShouldRunTestUtility + { + public static async Task Combine(params Func>[] getResults) + { + foreach (var getResult in getResults) + { + var result = await getResult(); + if (result != ShouldRunTestResult.Yes) + { + return result; + } + } + + return ShouldRunTestResult.Yes; + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/TestResult.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/TestResult.cs new file mode 100644 index 000000000..00f58c91f --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/TestResult.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// The possible outcomes of an . + /// + public enum TestResult + { + /// + /// The test completed successfully. + /// + Pass, + + /// + /// The test failed and should be investigated. + /// + Fail, + + /// + /// The test was skipped ( returned ). + /// + Skip, + + /// + /// The result of the test could not be determined ( returned ). + /// + /// + /// This result is returned if there is a newer catalog entry for this package. When that catalog entry is processed, a different result should be returned. + /// + Pending, + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/ValidationResult.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/ValidationResult.cs new file mode 100644 index 000000000..8933ec2a2 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Result/ValidationResult.cs @@ -0,0 +1,53 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Stores information about the outcome of a . + /// + public class ValidationResult + { + /// + /// The test that was run. + /// + [JsonProperty("validator")] + public IValidatorIdentity Validator { get; } + + /// + /// The result of the test. + /// + [JsonProperty("result")] + public TestResult Result { get; } + + /// + /// If the test ed, the exception that was thrown. + /// + [JsonProperty("exception")] + [JsonConverter(typeof(SafeExceptionConverter))] + public Exception Exception { get; } + + public ValidationResult(IValidator validator, TestResult result) + : this(validator, result, null) + { + } + + public ValidationResult(IValidatorIdentity validator, TestResult result, Exception exception) + { + Validator = validator; + Result = result; + Exception = exception; + } + + [JsonConstructor] + public ValidationResult(ValidatorIdentity validator, TestResult result, Exception exception) + { + Validator = validator; + Result = result; + Exception = exception; + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/AggregateValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/AggregateValidator.cs new file mode 100644 index 000000000..b332985c9 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/AggregateValidator.cs @@ -0,0 +1,49 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Abstract class with the shared functionality between all implementations. + /// + public abstract class AggregateValidator : IAggregateValidator + { + protected readonly IEnumerable Validators; + + protected readonly ILogger Logger; + + [JsonProperty("name")] + public virtual string Name + { + get + { + return GetType().FullName; + } + } + + public AggregateValidator(IEnumerable validators, ILogger logger) + { + Validators = validators ?? throw new ArgumentNullException(nameof(validators)); + Logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + /// Runs validations returned by . + /// + /// The to validate. + /// A which contains the results of the validation. + public async Task ValidateAsync(ValidationContext context) + { + return new AggregateValidationResult( + this, + await Task.WhenAll(Validators.Select(v => v.ValidateAsync(context)))); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Catalog/PackageHasSignatureValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Catalog/PackageHasSignatureValidator.cs new file mode 100644 index 000000000..9cf61c97f --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Catalog/PackageHasSignatureValidator.cs @@ -0,0 +1,130 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json.Linq; +using NuGet.Packaging.Signing; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Validates that the package is signed by verifying the presence of the "package signature file" + /// in the nupkg. See: https://github.com/NuGet/Home/wiki/Package-Signatures-Technical-Details#-the-package-signature-file + /// + public sealed class PackageHasSignatureValidator : Validator + { + private readonly ILogger _logger; + + public PackageHasSignatureValidator( + CatalogEndpoint endpoint, + ValidatorConfiguration config, + ILogger logger) + : base(endpoint, config, logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + protected override Task ShouldRunAsync(ValidationContext context) + { + return ShouldRunTestUtility.Combine( + () => Task.FromResult(ShouldRunValidator(context)), + () => base.ShouldRunAsync(context)); + } + + protected override async Task RunInternalAsync(ValidationContext context) + { + await RunValidatorAsync(context); + } + + public ShouldRunTestResult ShouldRunValidator(ValidationContext context) + { + if (!Config.RequireRepositorySignature) + { + return ShouldRunTestResult.No; + } + + var latest = context.Entries? + .OrderByDescending(e => e.CommitTimeStamp) + .FirstOrDefault(); + + if (latest == null) + { + _logger.LogInformation( + "Skipping package {PackageId} {PackageVersion} as it had no catalog entries", + context.Package.Id, + context.Package.Version); + + return ShouldRunTestResult.No; + } + + // We don't need to validate the package if the latest entry indicates deletion. + if (latest.IsDelete) + { + _logger.LogInformation( + "Skipping package {PackageId} {PackageVersion} as its latest catalog entry is a delete", + context.Package.Id, + context.Package.Version); + + return ShouldRunTestResult.No; + } + + return ShouldRunTestResult.Yes; + } + + public async Task RunValidatorAsync(ValidationContext context) + { + var latest = context.Entries + .OrderByDescending(e => e.CommitTimeStamp) + .FirstOrDefault(); + + _logger.LogInformation( + "Validating that catalog entry {CatalogEntry} for package {PackageId} {PackageVersion} has a package signature file...", + latest.Uri, + context.Package.Id, + context.Package.Version); + + var leaf = await context.Client.GetJObjectAsync(latest.Uri, context.CancellationToken); + + if (!HasSignatureFile(leaf, latest.Uri)) + { + _logger.LogWarning( + "Catalog entry {CatalogEntry} for package {PackageId} {PackageVersion} is missing a package signature file.", + latest.Uri, + context.Package.Id, + context.Package.Version); + + throw new MissingPackageSignatureFileException( + latest.Uri, + $"Catalog entry {latest.Uri} for package {context.Package.Id} {context.Package.Version} is missing a package signature file."); + } + + _logger.LogInformation( + "Validated that catalog entry {CatalogEntry} for package {PackageId} {PackageVersion} has a package signature.", + latest.Uri, + context.Package.Id, + context.Package.Version); + } + + private bool HasSignatureFile(JObject leaf, Uri uri) + { + const string propertyName = "packageEntries"; + + var packageEntries = leaf[propertyName]; + + if (packageEntries == null) + { + throw new InvalidOperationException($"The catalog leaf at {uri.AbsoluteUri} is missing the '{propertyName}' property."); + } + + if (!(packageEntries is JArray files)) + { + throw new InvalidOperationException($"The catalog leaf at {uri.AbsoluteUri} has a malformed '{propertyName}' property."); + } + + return files.Any(file => (string)file["fullName"] == SigningSpecifications.V1.SignaturePath); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/AggregateEndpointCursor.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/AggregateEndpointCursor.cs new file mode 100644 index 000000000..8c7d1884b --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/AggregateEndpointCursor.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// An based on a list of s. + /// + public class AggregateEndpointCursor : AggregateCursor + { + public AggregateEndpointCursor(IEnumerable endpoints) + : base(endpoints.Select(e => e.Cursor)) + { + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/CatalogEndpoint.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/CatalogEndpoint.cs new file mode 100644 index 000000000..d659f3abf --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/CatalogEndpoint.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Represents the catalog blobs endpoint, which is the transaction log that all of V3 is based off. + /// + /// + /// Validations associated with this class should be primarily based off or . + /// + public class CatalogEndpoint : IEndpoint + { + /// + /// Technically, the catalog has a cursor, but we are using the max value to represent it because if a catalog entry exists, then it must be able to be validated against. + /// We could fetch the cursor from the catalog and then verify that all catalog entries are before it, but that would be unnecessary. + /// + public ReadCursor Cursor => new MemoryCursor(MemoryCursor.MaxValue); + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/Endpoint.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/Endpoint.cs new file mode 100644 index 000000000..33095dcd4 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/Endpoint.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public abstract class Endpoint : IEndpoint + { + public Endpoint( + Uri cursorUri, + Func messageHandlerFactory) + { + Cursor = new HttpReadCursor(cursorUri, messageHandlerFactory); + } + + public ReadCursor Cursor { get; } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/EndpointConfiguration.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/EndpointConfiguration.cs new file mode 100644 index 000000000..4256a6de9 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/EndpointConfiguration.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// The config passed to s. + /// + public sealed class EndpointConfiguration + { + public EndpointConfiguration( + Uri registrationCursorUri, + Uri flatContainerCursorUri, + IReadOnlyDictionary instanceNameToSearchConfig) + { + RegistrationCursorUri = registrationCursorUri ?? throw new ArgumentNullException(nameof(registrationCursorUri)); + FlatContainerCursorUri = flatContainerCursorUri ?? throw new ArgumentNullException(nameof(flatContainerCursorUri)); + InstanceNameToSearchConfiguration = instanceNameToSearchConfig ?? throw new ArgumentNullException(nameof(instanceNameToSearchConfig)); + } + + public Uri RegistrationCursorUri { get; } + public Uri FlatContainerCursorUri { get; } + public IReadOnlyDictionary InstanceNameToSearchConfiguration { get; } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/EndpointValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/EndpointValidator.cs new file mode 100644 index 000000000..2c986d06c --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/EndpointValidator.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Runs a set of s. + /// + /// An that the s ran by this must be associated with. + public class EndpointValidator : AggregateValidator where T : class, IEndpoint + { + public EndpointValidator( + T endpoint, + IEnumerable> validators, + ILogger logger) : + base(validators, logger) + { + Endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint)); + } + + private T Endpoint { get; } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/FlatcontainerEndpoint.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/FlatcontainerEndpoint.cs new file mode 100644 index 000000000..1e1f92e4d --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/FlatcontainerEndpoint.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Represents the flat-container blobs endpoint, which stores nupkgs for packages and a directory of versions. + /// + public class FlatContainerEndpoint : Endpoint + { + public FlatContainerEndpoint( + EndpointConfiguration config, + Func messageHandlerFactory) + : base(config.FlatContainerCursorUri, messageHandlerFactory) + { + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/IEndpoint.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/IEndpoint.cs new file mode 100644 index 000000000..93f2a9bfe --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/IEndpoint.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Represents an endpoint, or V3 resource, to run validations against. + /// + public interface IEndpoint + { + /// + /// Used to derive the most recent catalog entry for which validations against this endpoint should be ran. + /// + ReadCursor Cursor { get; } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/RegistrationEndpoint.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/RegistrationEndpoint.cs new file mode 100644 index 000000000..adce2fe9b --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/RegistrationEndpoint.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Represents the registration blobs endpoint, which stores metadata about packages. + /// + public class RegistrationEndpoint : Endpoint + { + public RegistrationEndpoint( + EndpointConfiguration config, + Func messageHandlerFactory) + : base(config.RegistrationCursorUri, messageHandlerFactory) + { + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/SearchEndpoint.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/SearchEndpoint.cs new file mode 100644 index 000000000..2e4da9d50 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/SearchEndpoint.cs @@ -0,0 +1,32 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Represents the search endpoint, which allows querying for packages using keywords, by prefix (autocomplete), + /// and hijack. + /// + public class SearchEndpoint : IEndpoint + { + public SearchEndpoint( + string instanceName, + IReadOnlyList cursorUris, + Uri baseUri, + Func messageHandlerFactory) + { + Cursor = new AggregateCursor(cursorUris.Select(c => new HttpReadCursor(c, messageHandlerFactory))); + InstanceName = instanceName; + BaseUri = baseUri; + } + + public ReadCursor Cursor { get; } + public string InstanceName { get; } + public Uri BaseUri { get; } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/SearchEndpointConfiguration.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/SearchEndpointConfiguration.cs new file mode 100644 index 000000000..42fe13ccd --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Endpoint/SearchEndpointConfiguration.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public sealed class SearchEndpointConfiguration + { + public SearchEndpointConfiguration(IReadOnlyList cursorUris, Uri baseUri) + { + CursorUris = cursorUris; + BaseUri = baseUri; + } + + public IReadOnlyList CursorUris { get; } + public Uri BaseUri { get; } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/AggregateMetadataInconsistencyException.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/AggregateMetadataInconsistencyException.cs new file mode 100644 index 000000000..dda649799 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/AggregateMetadataInconsistencyException.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using System; +using System.Collections.Generic; + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Validation.Test.Exceptions +{ + public class AggregateMetadataInconsistencyException : MetadataInconsistencyException + { + public AggregateMetadataInconsistencyException(IReadOnlyCollection> innerExceptions) + : base(null) + { + InnerExceptions = innerExceptions ?? throw new ArgumentNullException(nameof(innerExceptions)); + Data.Add("InnerExceptions", JsonConvert.SerializeObject(InnerExceptions)); + } + + public IReadOnlyCollection> InnerExceptions { get; } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MetadataFieldInconsistencyException.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MetadataFieldInconsistencyException.cs new file mode 100644 index 000000000..40745a001 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MetadataFieldInconsistencyException.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class MetadataFieldInconsistencyException : MetadataInconsistencyException + { + public MetadataFieldInconsistencyException(TMetadata databaseMetadata, TMetadata v3Metadata, string fieldName, Func getField) + : this(databaseMetadata, v3Metadata, fieldName, getField(databaseMetadata), getField(v3Metadata)) + { + } + + public MetadataFieldInconsistencyException(TMetadata databaseMetadata, TMetadata v3Metadata, string fieldName, object databaseField, object v3Field) + : base(databaseMetadata, v3Metadata, $"{fieldName} does not match!") + { + Data.Add($"{nameof(DatabaseMetadata)}.{fieldName}", JsonConvert.SerializeObject(databaseField, JsonSerializerUtility.SerializerSettings)); + Data.Add($"{nameof(V3Metadata)}.{fieldName}", JsonConvert.SerializeObject(v3Field, JsonSerializerUtility.SerializerSettings)); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MetadataInconsistencyException.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MetadataInconsistencyException.cs new file mode 100644 index 000000000..7a778416c --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MetadataInconsistencyException.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class MetadataInconsistencyException : ValidationException + { + public MetadataInconsistencyException(string additionalMessage) + : base("The metadata between the database and V3 is inconsistent!" + + (additionalMessage != null ? $" {additionalMessage}" : "")) + { + } + } + + public class MetadataInconsistencyException : MetadataInconsistencyException + { + public T DatabaseMetadata { get; private set; } + public T V3Metadata { get; private set; } + + public MetadataInconsistencyException(T databaseMetadata, T v3Metadata) + : this(databaseMetadata, v3Metadata, null) + { + } + + public MetadataInconsistencyException(T databaseMetadata, T v3Metadata, string additionalMessage) + : base(additionalMessage) + { + DatabaseMetadata = databaseMetadata; + V3Metadata = v3Metadata; + + Data.Add(nameof(DatabaseMetadata), JsonConvert.SerializeObject(DatabaseMetadata)); + Data.Add(nameof(V3Metadata), JsonConvert.SerializeObject(V3Metadata)); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MissingPackageSignatureFileException.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MissingPackageSignatureFileException.cs new file mode 100644 index 000000000..fd591e76a --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MissingPackageSignatureFileException.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// The exception thrown when a catalog entry is missing a package signature file. + /// This indicates that the package is not signed. + /// See: https://github.com/NuGet/Home/wiki/Package-Signatures-Technical-Details#-the-package-signature-file + /// + public sealed class MissingPackageSignatureFileException : ValidationException + { + public MissingPackageSignatureFileException(Uri catalogEntry, string message) + : base(message) + { + CatalogEntry = catalogEntry; + + Data.Add(nameof(CatalogEntry), catalogEntry); + } + + public Uri CatalogEntry { get; } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MissingRepositorySignatureException.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MissingRepositorySignatureException.cs new file mode 100644 index 000000000..f8c71eb03 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/MissingRepositorySignatureException.cs @@ -0,0 +1,44 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Thrown when a package that is expected to be repository signed is missing the signature. + /// + public class MissingRepositorySignatureException : Exception + { + public MissingRepositorySignatureException(string message, MissingRepositorySignatureReason reason) + : base(message) + { + Reason = reason; + + Data.Add(nameof(Reason), reason); + } + + public MissingRepositorySignatureReason Reason { get; } + } + + /// + /// The reason why a was thrown. + /// + public enum MissingRepositorySignatureReason + { + /// + /// The package isn't signed. + /// + Unsigned, + + /// + /// The package is signed with an unknown signature type. + /// + UnknownSignature, + + /// + /// The package is author signed but doesn't have a repository countersignature. + /// + AuthorSignedNoRepositoryCountersignature, + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/TimestampComparisonException.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/TimestampComparisonException.cs new file mode 100644 index 000000000..4fc31b0ac --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/TimestampComparisonException.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class TimestampComparisonException : ValidationException + { + public PackageTimestampMetadata TimestampDatabase { get; } + public PackageTimestampMetadata TimestampCatalog { get; } + + public TimestampComparisonException(PackageTimestampMetadata timestampDatabase, PackageTimestampMetadata timestampCatalog, string message) + : base(message) + { + TimestampDatabase = timestampDatabase; + TimestampCatalog = timestampCatalog; + + Data.Add(nameof(TimestampDatabase), JsonConvert.SerializeObject(TimestampDatabase)); + Data.Add(nameof(TimestampCatalog), JsonConvert.SerializeObject(TimestampCatalog)); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/ValidationException.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/ValidationException.cs new file mode 100644 index 000000000..52ed7f60b --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Exceptions/ValidationException.cs @@ -0,0 +1,28 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Base class for exceptions throw by . + /// + public class ValidationException : Exception + { + public ValidationException() + : base() + { + } + + public ValidationException(string message) + : base(message) + { + } + + public ValidationException(string message, Exception e) + : base(message, e) + { + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/FlatContainer/FlatContainerValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/FlatContainer/FlatContainerValidator.cs new file mode 100644 index 000000000..67aaae9fc --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/FlatContainer/FlatContainerValidator.cs @@ -0,0 +1,32 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// The base class for all Package Content (aka "flat container") validations. + /// + public abstract class FlatContainerValidator : Validator + { + public FlatContainerValidator( + FlatContainerEndpoint endpoint, + ValidatorConfiguration config, + ILogger logger) + : base(endpoint, config, logger) + { + } + + protected Uri GetV3PackageUri(ValidationContext context) + { + // Based off DownloadResourceV3 + // See: https://github.com/NuGet/NuGet.Client/blob/3803820961f4d61c06d07b179dab1d0439ec0d91/src/NuGet.Core/NuGet.Protocol/Resources/DownloadResourceV3.cs#L78 + var id = context.Package.Id.ToLowerInvariant(); + var version = context.Package.Version.ToNormalizedString().ToLowerInvariant(); + + return new Uri($"{Config.PackageBaseAddress}/{id}/{version}/{id}.{version}.nupkg"); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/FlatContainer/PackageIsRepositorySignedValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/FlatContainer/PackageIsRepositorySignedValidator.cs new file mode 100644 index 000000000..4334f0f5d --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/FlatContainer/PackageIsRepositorySignedValidator.cs @@ -0,0 +1,116 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Packaging; +using NuGet.Packaging.Signing; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Validates that the package is repository signed. + /// + public class PackageIsRepositorySignedValidator : FlatContainerValidator + { + public PackageIsRepositorySignedValidator( + FlatContainerEndpoint endpoint, + ValidatorConfiguration config, + ILogger logger) + : base(endpoint, config, logger) + { + } + + protected async override Task ShouldRunAsync(ValidationContext context) + { + if (!Config.RequireRepositorySignature) + { + return ShouldRunTestResult.No; + } + + return await ShouldRunTestUtility.Combine( + () => base.ShouldRunAsync(context), + () => PackageExistsAsync(context)); + } + + protected async override Task RunInternalAsync(ValidationContext context) + { + // Get the package's signature, if any. + var signature = await GetPrimarySignatureOrNullAsync(context); + + if (signature == null) + { + throw new MissingRepositorySignatureException( + $"Package {context.Package.Id} {context.Package.Version} is unsigned.", + MissingRepositorySignatureReason.Unsigned); + } + + // The repository signature can be the primary signature or the author signature's countersignature. + IRepositorySignature repositorySignature = null; + + switch (signature.Type) + { + case SignatureType.Repository: + repositorySignature = (RepositoryPrimarySignature)signature; + break; + + case SignatureType.Author: + repositorySignature = RepositoryCountersignature.GetRepositoryCountersignature(signature); + + if (repositorySignature == null) + { + throw new MissingRepositorySignatureException( + $"Package {context.Package.Id} {context.Package.Version} is author signed but not repository signed.", + MissingRepositorySignatureReason.AuthorSignedNoRepositoryCountersignature); + } + + break; + + default: + case SignatureType.Unknown: + throw new MissingRepositorySignatureException( + $"Package {context.Package.Id} {context.Package.Version} has an unknown signature type '{signature.Type}'.", + MissingRepositorySignatureReason.UnknownSignature); + } + + Logger.LogInformation( + "Package {PackageId} {PackageVersion} has a repository signature with service index {ServiceIndex} and owners {Owners}.", + context.Package.Id, + context.Package.Version, + repositorySignature.V3ServiceIndexUrl, + repositorySignature.PackageOwners); + } + + private async Task GetPrimarySignatureOrNullAsync(ValidationContext context) + { + var downloader = new PackageDownloader(context.Client, Logger); + var uri = GetV3PackageUri(context); + + using (var packageStream = await downloader.DownloadAsync(uri, context.CancellationToken)) + { + if (packageStream == null) + { + throw new InvalidOperationException($"Package {context.Package.Id} {context.Package.Version} couldn't be downloaded at {uri.AbsoluteUri}."); + } + + using (var package = new PackageArchiveReader(packageStream)) + { + return await package.GetPrimarySignatureAsync(context.CancellationToken); + } + } + } + + private async Task PackageExistsAsync(ValidationContext context) + { + var uri = GetV3PackageUri(context); + + using (var request = new HttpRequestMessage(HttpMethod.Head, uri)) + using (var response = await context.Client.SendAsync(request)) + { + return response.IsSuccessStatusCode ? ShouldRunTestResult.Yes : ShouldRunTestResult.No; + } + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IAggregateValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IAggregateValidator.cs new file mode 100644 index 000000000..ac5e57565 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IAggregateValidator.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Runs a set of s. + /// + public interface IAggregateValidator : IValidatorIdentity + { + /// + /// Runs each and returns an containing all results. + /// + Task ValidateAsync(ValidationContext context); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IValidator.cs new file mode 100644 index 000000000..c7153dde2 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IValidator.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Performs a validation test on a . + /// + public interface IValidator : IValidatorIdentity + { + /// + /// Validates a package. + /// + /// A which contains the results of the validation. + Task ValidateAsync(ValidationContext context); + } + + /// + /// Performs a validation test on a package on a . + /// + /// The to be validated. + public interface IValidator : IValidator where T : IEndpoint + { + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IValidatorIdentity.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IValidatorIdentity.cs new file mode 100644 index 000000000..f1b0907b2 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/IValidatorIdentity.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public interface IValidatorIdentity + { + /// + /// Human readable name that represents the identity of the validation that was run. + /// + string Name { get; } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidator.cs new file mode 100644 index 000000000..3d9ff0c33 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidator.cs @@ -0,0 +1,93 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Versioning; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Stores a set of s and runs them. + /// + public class PackageValidator + { + public IEnumerable AggregateValidators { get; } + + private readonly ValidationSourceRepositories _sourceRepositories; + private readonly ILogger _logger; + private readonly ILogger _contextLogger; + private readonly StorageFactory _auditingStorageFactory; + + public PackageValidator( + IEnumerable aggregateValidators, + StorageFactory auditingStorageFactory, + ValidationSourceRepositories sourceRepositories, + ILogger logger, + ILogger contextLogger) + { + var validators = aggregateValidators?.ToList(); + + if (aggregateValidators == null || !validators.Any()) + { + throw new ArgumentException("Must supply at least one endpoint!", nameof(aggregateValidators)); + } + + AggregateValidators = validators; + _auditingStorageFactory = auditingStorageFactory ?? throw new ArgumentNullException(nameof(auditingStorageFactory)); + _sourceRepositories = sourceRepositories ?? throw new ArgumentNullException(nameof(sourceRepositories)); + _logger = logger; + _contextLogger = contextLogger ?? throw new ArgumentNullException(nameof(contextLogger)); + } + + /// + /// Runs s from the s against a package. + /// + /// A generated from the results of the s. + public async Task ValidateAsync(PackageValidatorContext context, CollectorHttpClient client, CancellationToken cancellationToken) + { + if (context == null) + { + throw new ArgumentNullException(nameof(context)); + } + + if (client == null) + { + throw new ArgumentNullException(nameof(client)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var package = new PackageIdentity(context.Package.Id, NuGetVersion.Parse(context.Package.Version)); + + var deletionAuditEntries = await DeletionAuditEntry.GetAsync( + _auditingStorageFactory, + cancellationToken, + package, + logger: _logger); + + var validationContext = new ValidationContext( + package, + context.CatalogEntries, + deletionAuditEntries, + _sourceRepositories, + client, + cancellationToken, + _contextLogger); + + var results = await Task.WhenAll( + AggregateValidators + .Select(endpoint => endpoint.ValidateAsync(validationContext)) + .ToList()); + + return new PackageValidationResult(validationContext, results); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorContext.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorContext.cs new file mode 100644 index 000000000..40d0c8b54 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorContext.cs @@ -0,0 +1,46 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// The data to be passed to . + /// + public class PackageValidatorContext + { + /// + /// This should be incremented every time the structure of this class changes. + /// + public const int Version = 1; + + /// + /// The package to run validations on. + /// + public FeedPackageIdentity Package { get; } + + /// + /// The catalog entries that initiated this request to run validations. + /// + /// + /// If null, the latest catalog index entry for the package will be validated against. + /// + public IEnumerable CatalogEntries { get; } + + [JsonConstructor] + public PackageValidatorContext(FeedPackageIdentity package, IEnumerable catalogEntries) + { + Package = package ?? throw new ArgumentNullException(nameof(package)); + CatalogEntries = catalogEntries; + } + + public PackageValidatorContext(PackageMonitoringStatus status) + : this(status.Package, status.ValidationResult?.CatalogEntries) + { + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorContextEnqueuer.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorContextEnqueuer.cs new file mode 100644 index 000000000..ec00cf364 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/PackageValidatorContextEnqueuer.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Fetches and enqueues the s associated with recent catalog entries to be processed by . + /// + public class PackageValidatorContextEnqueuer + { + private readonly ValidationCollector _collector; + private readonly ReadWriteCursor _front; + private readonly ReadCursor _back; + + public PackageValidatorContextEnqueuer( + ValidationCollector collector, + ReadWriteCursor front, + ReadCursor back) + { + _collector = collector ?? throw new ArgumentNullException(nameof(collector)); + _front = front ?? throw new ArgumentNullException(nameof(front)); + _back = back ?? throw new ArgumentNullException(nameof(back)); + } + + public async Task EnqueuePackageValidatorContexts(CancellationToken token) + { + bool run; + do + { + run = await _collector.RunAsync(_front, _back, token); + } + while (run); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationDeprecationValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationDeprecationValidator.cs new file mode 100644 index 000000000..099ba51f3 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationDeprecationValidator.cs @@ -0,0 +1,136 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; +using NuGet.Services.Metadata.Catalog.Monitoring.Model; +using NuGet.Services.Metadata.Catalog.Monitoring.Validation.Test.Exceptions; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace NuGet.Services.Metadata.Catalog.Monitoring.Validation.Test.Registration +{ + public class RegistrationDeprecationValidator : RegistrationIndexValidator + { + public RegistrationDeprecationValidator( + RegistrationEndpoint endpoint, + ValidatorConfiguration config, + ILogger logger) + : base(endpoint, config, logger) + { + } + + public override Task CompareIndexAsync(ValidationContext context, PackageRegistrationIndexMetadata database, PackageRegistrationIndexMetadata v3) + { + var exceptions = new List>(); + + if (database.Deprecation == null && v3.Deprecation == null) + { + return Task.CompletedTask; + } + else if (database.Deprecation == null || v3.Deprecation == null) + { + throw new MetadataFieldInconsistencyException( + database, v3, + nameof(PackageRegistrationIndexMetadata.Deprecation), + i => i.Deprecation); + } + + if (!database.Deprecation.Reasons.OrderBy(r => r).SequenceEqual(v3.Deprecation.Reasons.OrderBy(r => r))) + { + AddDeprecationInconsistencyException( + exceptions, + database, v3, + nameof(PackageRegistrationDeprecationMetadata.Reasons), + d => d.Reasons); + } + + if (database.Deprecation.Message != v3.Deprecation.Message) + { + AddDeprecationInconsistencyException( + exceptions, + database, v3, + nameof(PackageRegistrationDeprecationMetadata.Message), + d => d.Message); + } + + CompareIndexAlternatePackage(exceptions, database, v3); + + if (exceptions.Any()) + { + throw new AggregateMetadataInconsistencyException(exceptions); + } + + return Task.CompletedTask; + } + + private void CompareIndexAlternatePackage( + List> exceptions, + PackageRegistrationIndexMetadata database, + PackageRegistrationIndexMetadata v3) + { + if (database.Deprecation.AlternatePackage == null && v3.Deprecation.AlternatePackage == null) + { + return; + } + else if (database.Deprecation.AlternatePackage == null || v3.Deprecation.AlternatePackage == null) + { + AddDeprecationInconsistencyException( + exceptions, + database, v3, + nameof(PackageRegistrationDeprecationMetadata.AlternatePackage), + d => d.AlternatePackage); + + return; + } + + if (database.Deprecation.AlternatePackage.Id != v3.Deprecation.AlternatePackage.Id) + { + AddAlternatePackageInconsistencyException( + exceptions, + database, v3, + nameof(PackageRegistrationAlternatePackageMetadata.Id), + a => a.Id); + } + + if (database.Deprecation.AlternatePackage.Range != v3.Deprecation.AlternatePackage.Range) + { + AddAlternatePackageInconsistencyException( + exceptions, + database, v3, + nameof(PackageRegistrationAlternatePackageMetadata.Range), + a => a.Range); + } + } + + private void AddDeprecationInconsistencyException( + List> list, + PackageRegistrationIndexMetadata database, + PackageRegistrationIndexMetadata v3, + string deprecationFieldName, + Func getDeprecationField) + { + var exception = new MetadataFieldInconsistencyException( + database, v3, + nameof(PackageRegistrationIndexMetadata.Deprecation) + "." + deprecationFieldName, + i => getDeprecationField(i.Deprecation)); + + list.Add(exception); + } + + private void AddAlternatePackageInconsistencyException( + List> list, + PackageRegistrationIndexMetadata database, + PackageRegistrationIndexMetadata v3, + string alternatePackageField, + Func getAlternatePackageField) + { + AddDeprecationInconsistencyException( + list, + database, v3, + nameof(PackageRegistrationDeprecationMetadata.AlternatePackage) + "." + alternatePackageField, + d => getAlternatePackageField(d.AlternatePackage)); + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationExistsValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationExistsValidator.cs new file mode 100644 index 000000000..188afb490 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationExistsValidator.cs @@ -0,0 +1,59 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class RegistrationExistsValidator : RegistrationLeafValidator + { + public RegistrationExistsValidator( + RegistrationEndpoint endpoint, + ValidatorConfiguration config, + ILogger logger) + : base(endpoint, config, logger) + { + } + + public override Task ShouldRunLeafAsync( + ValidationContext context, + PackageRegistrationLeafMetadata database, + PackageRegistrationLeafMetadata v3) + { + return Task.FromResult(ShouldRunTestResult.Yes); + } + + public override Task CompareLeafAsync( + ValidationContext context, + PackageRegistrationLeafMetadata database, + PackageRegistrationLeafMetadata v3) + { + var databaseExists = database != null; + var v3Exists = v3 != null; + var completedTask = Task.FromResult(0); + + if (databaseExists != v3Exists) + { + // Currently, leaf nodes are not deleted after a package is deleted. + // This is a known bug. Do not fail validations because of it. + // See https://github.com/NuGet/NuGetGallery/issues/4475 + if (v3Exists && !(v3 is PackageRegistrationIndexMetadata)) + { + Logger.LogInformation("{PackageId} {PackageVersion} doesn't exist in the database but has a leaf node in V3!", context.Package.Id, context.Package.Version); + return completedTask; + } + + const string existsString = "exists"; + const string doesNotExistString = "doesn't exist"; + + throw new MetadataInconsistencyException( + database, + v3, + $"Database {(databaseExists ? existsString : doesNotExistString)} but V3 {(v3Exists ? existsString : doesNotExistString)}!"); + } + + return completedTask; + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationIdValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationIdValidator.cs new file mode 100644 index 000000000..bb1ecf50c --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationIdValidator.cs @@ -0,0 +1,35 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class RegistrationIdValidator : RegistrationIndexValidator + { + public RegistrationIdValidator( + RegistrationEndpoint endpoint, + ValidatorConfiguration config, + ILogger logger) + : base(endpoint, config, logger) + { + } + + public override Task CompareIndexAsync( + ValidationContext context, + PackageRegistrationIndexMetadata database, + PackageRegistrationIndexMetadata v3) + { + if (!database.Id.Equals(v3.Id, System.StringComparison.OrdinalIgnoreCase)) + { + throw new MetadataFieldInconsistencyException( + database, v3, + nameof(PackageRegistrationIndexMetadata.Id), + m => m.Id); + } + + return Task.FromResult(0); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationIndexValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationIndexValidator.cs new file mode 100644 index 000000000..879692c64 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationIndexValidator.cs @@ -0,0 +1,58 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public abstract class RegistrationIndexValidator : RegistrationValidator + { + public RegistrationIndexValidator( + RegistrationEndpoint endpoint, + ValidatorConfiguration config, + ILogger logger) + : base(endpoint, config, logger) + { + } + + protected override async Task ShouldRunAsync(ValidationContext context) + { + var databaseIndex = await context.GetIndexDatabaseAsync(); + var v3Index = await context.GetIndexV3Async(); + + return await ShouldRunTestUtility.Combine( + () => base.ShouldRunAsync(context), + () => ShouldRunIndexAsync(context, databaseIndex, v3Index)); + } + + protected override async Task RunInternalAsync(ValidationContext context) + { + var databaseIndex = await context.GetIndexDatabaseAsync(); + var v3Index = await context.GetIndexV3Async(); + + try + { + await CompareIndexAsync(context, databaseIndex, v3Index); + } + catch (Exception e) + { + throw new ValidationException("Registration index metadata does not match the database metadata!", e); + } + } + + public Task ShouldRunIndexAsync( + ValidationContext context, + PackageRegistrationIndexMetadata database, + PackageRegistrationIndexMetadata v3) + { + return Task.FromResult(database != null && v3 != null ? ShouldRunTestResult.Yes : ShouldRunTestResult.No); + } + + public abstract Task CompareIndexAsync( + ValidationContext context, + PackageRegistrationIndexMetadata database, + PackageRegistrationIndexMetadata v3); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationLeafValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationLeafValidator.cs new file mode 100644 index 000000000..28a3400f9 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationLeafValidator.cs @@ -0,0 +1,79 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public abstract class RegistrationLeafValidator : RegistrationValidator + { + public RegistrationLeafValidator( + RegistrationEndpoint endpoint, + ValidatorConfiguration config, + ILogger logger) + : base(endpoint, config, logger) + { + } + + protected override async Task ShouldRunAsync(ValidationContext context) + { + var databaseIndex = await context.GetIndexDatabaseAsync(); + var v3Index = await context.GetIndexV3Async(); + var databaseLeaf = await context.GetLeafDatabaseAsync(); + var v3Leaf = await context.GetLeafV3Async(); + + return await ShouldRunTestUtility.Combine( + () => base.ShouldRunAsync(context), + () => ShouldRunLeafAsync(context, databaseIndex, v3Index), + () => ShouldRunLeafAsync(context, databaseLeaf, v3Leaf)); + } + + protected override async Task RunInternalAsync(ValidationContext context) + { + var exceptions = new List(); + + var databaseIndex = await context.GetIndexDatabaseAsync(); + var v3Index = await context.GetIndexV3Async(); + + try + { + await CompareLeafAsync(context, databaseIndex, v3Index); + } + catch (Exception e) + { + exceptions.Add(new ValidationException("Registration index metadata does not match the database!", e)); + } + + var databaseLeaf = await context.GetLeafDatabaseAsync(); + var v3Leaf = await context.GetLeafV3Async(); + + try + { + await CompareLeafAsync(context, databaseLeaf, v3Leaf); + } + catch (Exception e) + { + exceptions.Add(new ValidationException("Registration leaf metadata does not match the database!", e)); + } + + if (exceptions.Any()) + { + throw new AggregateException(exceptions); + } + } + + public abstract Task ShouldRunLeafAsync( + ValidationContext context, + PackageRegistrationLeafMetadata database, + PackageRegistrationLeafMetadata v3); + + public abstract Task CompareLeafAsync( + ValidationContext context, + PackageRegistrationLeafMetadata database, + PackageRegistrationLeafMetadata v3); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationListedValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationListedValidator.cs new file mode 100644 index 000000000..a1b590dd4 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationListedValidator.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class RegistrationListedValidator : RegistrationLeafValidator + { + public RegistrationListedValidator( + RegistrationEndpoint endpoint, + ValidatorConfiguration config, + ILogger logger) + : base(endpoint, config, logger) + { + } + + public override Task ShouldRunLeafAsync( + ValidationContext context, + PackageRegistrationLeafMetadata database, + PackageRegistrationLeafMetadata v3) + { + return Task.FromResult(database != null && v3 != null ? ShouldRunTestResult.Yes : ShouldRunTestResult.No); + } + + public override Task CompareLeafAsync( + ValidationContext context, + PackageRegistrationLeafMetadata database, + PackageRegistrationLeafMetadata v3) + { + if (database.Listed != v3.Listed) + { + throw new MetadataFieldInconsistencyException( + database, v3, + nameof(PackageRegistrationLeafMetadata.Listed), + m => m.Listed); + } + + return Task.FromResult(0); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationRequireLicenseAcceptanceValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationRequireLicenseAcceptanceValidator.cs new file mode 100644 index 000000000..1665638cb --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationRequireLicenseAcceptanceValidator.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class RegistrationRequireLicenseAcceptanceValidator : RegistrationIndexValidator + { + public RegistrationRequireLicenseAcceptanceValidator( + RegistrationEndpoint endpoint, + ValidatorConfiguration config, + ILogger logger) + : base(endpoint, config, logger) + { + } + + public override Task CompareIndexAsync(ValidationContext context, PackageRegistrationIndexMetadata database, PackageRegistrationIndexMetadata v3) + { + var isEqual = database.RequireLicenseAcceptance == v3.RequireLicenseAcceptance; + + if (!isEqual) + { + throw new MetadataFieldInconsistencyException( + database, v3, + nameof(PackageRegistrationIndexMetadata.RequireLicenseAcceptance), + m => m.RequireLicenseAcceptance); + } + + return Task.FromResult(0); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationValidator.cs new file mode 100644 index 000000000..fa831cde5 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationValidator.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public abstract class RegistrationValidator : Validator + { + public RegistrationValidator( + RegistrationEndpoint endpoint, + ValidatorConfiguration config, + ILogger logger) + : base(endpoint, config, logger) + { + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationVersionValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationVersionValidator.cs new file mode 100644 index 000000000..0eb4b4c2e --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Registration/RegistrationVersionValidator.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class RegistrationVersionValidator : RegistrationIndexValidator + { + public RegistrationVersionValidator( + RegistrationEndpoint endpoint, + ValidatorConfiguration config, + ILogger logger) + : base(endpoint, config, logger) + { + } + + public override Task CompareIndexAsync(ValidationContext context, PackageRegistrationIndexMetadata database, PackageRegistrationIndexMetadata v3) + { + var isEqual = database.Version == v3.Version; + + if (!isEqual) + { + throw new MetadataFieldInconsistencyException( + database, v3, + nameof(PackageRegistrationIndexMetadata.Version), + m => m.Version.ToFullString()); + } + + return Task.FromResult(0); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Search/SearchHasVersionValidator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Search/SearchHasVersionValidator.cs new file mode 100644 index 000000000..f2552497c --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Search/SearchHasVersionValidator.cs @@ -0,0 +1,72 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class SearchHasVersionValidator : Validator + { + public SearchHasVersionValidator( + SearchEndpoint endpoint, + ValidatorConfiguration config, + ILogger logger) + : base(endpoint, config, logger) + { + } + + protected override async Task RunInternalAsync(ValidationContext context) + { + var searchVisible = await IsVisibleInSearchAsync(context); + var databaseState = await GetDatabaseStateAsync(context); + var databaseVisible = databaseState == DatabaseState.Listed; + if (databaseVisible != searchVisible) + { + const string listedString = "listed"; + const string unlistedString = "unlisted"; + + throw new MetadataInconsistencyException( + $"Database shows {databaseState.ToString().ToLowerInvariant()}" + + $" but search shows {(searchVisible ? listedString : unlistedString)}."); + } + } + + private async Task IsVisibleInSearchAsync(ValidationContext context) + { + var searchPage = await context.GetSearchPageForIdAsync(Endpoint.BaseUri); + var searchItem = searchPage.SingleOrDefault(); + bool searchListed; + if (searchItem != null) + { + var searchVersions = await searchItem.GetVersionsAsync(); + searchListed = searchVersions.Any(x => x.Version == context.Package.Version); + } + else + { + searchListed = false; + } + + return searchListed; + } + + private static async Task GetDatabaseStateAsync(ValidationContext context) + { + var databaseResult = await context.GetIndexDatabaseAsync(); + if (databaseResult == null) + { + return DatabaseState.Unavailable; + } + + return databaseResult.Listed ? DatabaseState.Listed : DatabaseState.Unlisted; + } + + private enum DatabaseState + { + Unavailable, + Unlisted, + Listed, + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationContext.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationContext.cs new file mode 100644 index 000000000..f0d6f425c --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationContext.cs @@ -0,0 +1,148 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NuGet.Packaging.Core; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Contains context for when a test is run. + /// + public class ValidationContext + { + private readonly IPackageRegistrationMetadataResource _databasePackageRegistrationMetadataResource; + private readonly IPackageRegistrationMetadataResource _v3PackageRegistrationMetadataResource; + private readonly HttpSourceResource _httpSourceResource; + private readonly Lazy> _databaseIndex; + private readonly Lazy> _v3Index; + private readonly Lazy> _databaseLeaf; + private readonly Lazy> _v3Leaf; + private readonly IPackageTimestampMetadataResource _databasetimestampMetadataResource; + private readonly Lazy> _timestampMetadataDatabase; + private readonly ILogger _logger; + private readonly Common.ILogger _commonLogger; + + private readonly ConcurrentDictionary>>> _searchPageForIdCache + = new ConcurrentDictionary>>>(); + + /// + /// The to run the test on. + /// + public PackageIdentity Package { get; } + + /// + /// The s for the package that were collected. + /// + /// + /// This can be null. + /// Most validations are queued with the catalog entries of a package, + /// but sometimes we do not know the catalog entries associated with a package but still want to run validations against the current state of V3. + /// This could happen if a change was never ingested by V3 and there is no catalog entry associated the package. + /// It could also happen if we lose the catalog entries of a package due to a message processing failure (). + /// + public IReadOnlyList Entries { get; } + + /// + /// The s, if any are associated with the . + /// + public IReadOnlyList DeletionAuditEntries { get; } + + /// + /// The to use when needed. + /// + public CollectorHttpClient Client { get; } + + /// + /// A associated with this run of the test. + /// + public CancellationToken CancellationToken { get; } + + public ValidationContext( + PackageIdentity package, + IEnumerable entries, + IEnumerable deletionAuditEntries, + ValidationSourceRepositories sourceRepositories, + CollectorHttpClient client, + CancellationToken token, + ILogger logger) + { + if (deletionAuditEntries == null) + { + throw new ArgumentNullException(nameof(deletionAuditEntries)); + } + + if (sourceRepositories == null) + { + throw new ArgumentNullException(nameof(sourceRepositories)); + } + + Package = package ?? throw new ArgumentNullException(nameof(package)); + Entries = entries?.ToList(); + DeletionAuditEntries = deletionAuditEntries.ToList(); + Client = client ?? throw new ArgumentNullException(nameof(client)); + CancellationToken = token; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _commonLogger = logger.AsCommon(); + + _databasetimestampMetadataResource = sourceRepositories.V2.GetResource(); + _databasePackageRegistrationMetadataResource = sourceRepositories.V2.GetResource(); + _v3PackageRegistrationMetadataResource = sourceRepositories.V3.GetResource(); + _httpSourceResource = sourceRepositories.V3.GetResource(); + + _databaseIndex = new Lazy>( + () => _databasePackageRegistrationMetadataResource.GetIndexAsync(Package, _commonLogger, CancellationToken)); + _v3Index = new Lazy>( + () => _v3PackageRegistrationMetadataResource.GetIndexAsync(Package, _commonLogger, CancellationToken)); + + _databaseLeaf = new Lazy>( + () => _databasePackageRegistrationMetadataResource.GetLeafAsync(Package, _commonLogger, CancellationToken)); + _v3Leaf = new Lazy>( + () => _v3PackageRegistrationMetadataResource.GetLeafAsync(Package, _commonLogger, CancellationToken)); + + _timestampMetadataDatabase = new Lazy>( + () => _databasetimestampMetadataResource.GetAsync(this)); + } + + public async Task> GetSearchPageForIdAsync(Uri searchBaseUri) + { + var lazyTask = _searchPageForIdCache.GetOrAdd( + searchBaseUri, + _ => new Lazy>>(() => + { + var queryUri = new Uri(searchBaseUri, "/query"); + _logger.LogInformation( + "Searching for package {Id} on {Uri}.", + Package.Id, + queryUri.AbsoluteUri); + + var rawSearchResource = new RawSearchResourceV3(_httpSourceResource.HttpSource, new[] { queryUri }); + var packageSearchResource = new PackageSearchResourceV3(rawSearchResource); + return packageSearchResource.SearchAsync( + searchTerm: $"packageid:{Package.Id}", + filter: new SearchFilter(includePrerelease: true), + skip: 0, + take: 1, + log: _commonLogger, + cancellationToken: CancellationToken); + })); + + return await lazyTask.Value; + } + + public Task GetIndexDatabaseAsync() => _databaseIndex.Value; + public Task GetIndexV3Async() => _v3Index.Value; + public Task GetLeafDatabaseAsync() => _databaseLeaf.Value; + public Task GetLeafV3Async() => _v3Leaf.Value; + public Task GetTimestampMetadataDatabaseAsync() => _timestampMetadataDatabase.Value; + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationSourceRepositories.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationSourceRepositories.cs new file mode 100644 index 000000000..bbcc04dfd --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationSourceRepositories.cs @@ -0,0 +1,32 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Protocol.Core.Types; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Exposes NuGet client APIs to be used in validations. + /// + public class ValidationSourceRepositories + { + public ValidationSourceRepositories( + SourceRepository v2, + SourceRepository v3) + { + V2 = v2 ?? throw new ArgumentNullException(nameof(v2)); + V3 = v3 ?? throw new ArgumentNullException(nameof(v3)); + } + + /// + /// The that can be used to get V2 resources. + /// + public SourceRepository V2 { get; } + + /// + /// The that can be used to get V3 resources. + /// + public SourceRepository V3 { get; } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationSourceRepository.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationSourceRepository.cs new file mode 100644 index 000000000..f26cf99ab --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidationSourceRepository.cs @@ -0,0 +1,32 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using NuGet.Configuration; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// This class exists to add some extra logic to the construction of using DI. + /// + public class ValidationSourceRepository : SourceRepository + { + public ValidationSourceRepository( + PackageSource source, + IEnumerable> lazyProviders, + IEnumerable providers, + FeedType type) + : base( + source, + providers + .Select(p => new Lazy(() => p)) + .Concat(lazyProviders), + type) + { + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Validator.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Validator.cs new file mode 100644 index 000000000..2c76d894d --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/Validator.cs @@ -0,0 +1,126 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Abstract class with the shared functionality between all implementations. + /// + public abstract class Validator : IValidator + { + protected readonly ValidatorConfiguration Config; + protected readonly ILogger Logger; + protected readonly Common.ILogger CommonLogger; + + public virtual string Name + { + get + { + return GetType().FullName; + } + } + + protected Validator(ValidatorConfiguration config, ILogger logger) + { + Config = config ?? throw new ArgumentNullException(nameof(config)); + Logger = logger ?? throw new ArgumentNullException(nameof(logger)); + CommonLogger = logger.AsCommon(); + } + + public async Task ValidateAsync(ValidationContext context) + { + try + { + ShouldRunTestResult shouldRun; + try + { + shouldRun = await ShouldRunAsync(context); + } + catch (Exception e) + { + throw new ValidationException("Threw an exception while trying to determine whether or not validation should run!", e); + } + + switch (shouldRun) + { + case ShouldRunTestResult.Yes: + await RunInternalAsync(context); + break; + case ShouldRunTestResult.No: + return new ValidationResult(this, TestResult.Skip); + case ShouldRunTestResult.RetryLater: + return new ValidationResult(this, TestResult.Pending); + } + } + catch (Exception e) + { + return new ValidationResult(this, TestResult.Fail, e); + } + + return new ValidationResult(this, TestResult.Pass); + } + + /// + /// Checks that the current batch of catalog entries contains the entry that was created from the current state of the database. + /// + /// + /// Our validations depend on the fact that the database and V3 are expected to have the same version of a package. + /// If the catalog entry we're running validations on, which is supposed to represent the current state of V3, is less recent than the database, then we shouldn't run validations. + /// + protected virtual async Task ShouldRunAsync(ValidationContext context) + { + if (context.Entries == null) + { + // If we don't have any catalog entries to use to compare timestamps, assume the database and V3 are in the same state and run validations anyway. + return ShouldRunTestResult.Yes; + } + + var timestampDatabase = await context.GetTimestampMetadataDatabaseAsync(); + var timestampCatalog = await PackageTimestampMetadata.FromCatalogEntries(context.Client, context.Entries); + + if (!timestampDatabase.Last.HasValue) + { + throw new TimestampComparisonException(timestampDatabase, timestampCatalog, + "Cannot get timestamp data for package from the database!"); + } + + if (!timestampCatalog.Last.HasValue) + { + throw new TimestampComparisonException(timestampDatabase, timestampCatalog, + "Cannot get timestamp data for package from the catalog!"); + } + + if (timestampCatalog.Last > timestampDatabase.Last) + { + throw new TimestampComparisonException(timestampDatabase, timestampCatalog, + "The timestamp in the catalog is newer than the timestamp in the database! This should never happen because all data flows from the feed into the catalog!"); + } + + return timestampCatalog.Last == timestampDatabase.Last + // If the timestamp metadata in the catalog is EQUAL to that of the database, we are looking at the latest catalog entry that corresponds with this package, so run the test. + ? ShouldRunTestResult.Yes + // If the timestamp metadata in the catalog is LESS than that of the database, we must not be looking at the latest entry that corresponds with this package, so we must attempt this test again later with more information. + : ShouldRunTestResult.RetryLater; + } + + protected abstract Task RunInternalAsync(ValidationContext context); + } + + /// + /// Abstract class with the shared functionality between all implementations. + /// + public abstract class Validator : Validator, IValidator where T : class, IEndpoint + { + protected Validator(T endpoint, ValidatorConfiguration config, ILogger logger) + : base(config, logger) + { + Endpoint = endpoint ?? throw new ArgumentNullException(nameof(endpoint)); + } + + public T Endpoint { get; } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidatorConfiguration.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidatorConfiguration.cs new file mode 100644 index 000000000..de3266ab0 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidatorConfiguration.cs @@ -0,0 +1,35 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// The config passed to s. + /// + public sealed class ValidatorConfiguration + { + public ValidatorConfiguration(string packageBaseAddress, bool requireRepositorySignature) + { + if (string.IsNullOrEmpty(packageBaseAddress)) + { + throw new ArgumentException("Package base address is required", nameof(packageBaseAddress)); + } + + PackageBaseAddress = packageBaseAddress.TrimEnd('/'); + RequireRepositorySignature = requireRepositorySignature; + } + + /// + /// The base URL for the Package Content resource. + /// See: https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource + /// + public string PackageBaseAddress { get; } + + /// + /// Whether repository signature validations are required. + /// + public bool RequireRepositorySignature { get; } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidatorIdentity.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidatorIdentity.cs new file mode 100644 index 000000000..135992071 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/Test/ValidatorIdentity.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + public class ValidatorIdentity : IValidatorIdentity + { + [JsonProperty("name")] + public string Name { get; } + + [JsonConstructor] + public ValidatorIdentity(string name) + { + Name = name; + } + } +} diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/ValidationCollector.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/ValidationCollector.cs new file mode 100644 index 000000000..d89471ef5 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/ValidationCollector.cs @@ -0,0 +1,59 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json.Linq; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Storage; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Creates s from Catalog entries and adds them to a s. + /// + public class ValidationCollector : SortingIdVersionCollector + { + private readonly IStorageQueue _queue; + + private ILogger _logger; + + public ValidationCollector( + IStorageQueue queue, + Uri index, + ITelemetryService telemetryService, + ILogger logger, + Func handlerFunc = null) + : base(index, telemetryService, handlerFunc) + { + _queue = queue; + _logger = logger; + } + + protected override async Task ProcessSortedBatchAsync( + CollectorHttpClient client, + KeyValuePair> sortedBatch, + JToken context, + CancellationToken cancellationToken) + { + var packageId = sortedBatch.Key.Id; + var packageVersion = sortedBatch.Key.Version; + var feedPackage = new FeedPackageIdentity(packageId, packageVersion); + + _logger.LogInformation("Processing catalog entries for {PackageId} {PackageVersion}.", packageId, packageVersion); + + var catalogEntries = sortedBatch.Value.Select(CatalogIndexEntry.Create); + + _logger.LogInformation("Adding {MostRecentCatalogEntryUri} to queue.", catalogEntries.OrderByDescending(c => c.CommitTimeStamp).First().Uri); + + await _queue.AddAsync( + new PackageValidatorContext(feedPackage, catalogEntries), + cancellationToken); + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/ValidationFactory.cs b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/ValidationFactory.cs new file mode 100644 index 000000000..57c0cffe8 --- /dev/null +++ b/src/NuGet.Services.Metadata.Catalog.Monitoring/Validation/ValidationFactory.cs @@ -0,0 +1,141 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using Autofac; +using Autofac.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Storage; +using StorageFactory = NuGet.Services.Metadata.Catalog.Persistence.StorageFactory; + +namespace NuGet.Services.Metadata.Catalog.Monitoring +{ + /// + /// Helper class for constructing validation business logic implementations. + /// + public static class ValidationFactory + { + public static PackageValidator CreatePackageValidator( + string galleryUrl, + string indexUrl, + StorageFactory auditingStorageFactory, + ValidatorConfiguration validatorConfig, + EndpointConfiguration endpointConfig, + Func messageHandlerFactory, + IGalleryDatabaseQueryService galleryDatabase, + ILoggerFactory loggerFactory) + { + if (auditingStorageFactory == null) + { + throw new ArgumentNullException(nameof(auditingStorageFactory)); + } + + var collection = new ServiceCollection(); + collection.AddSingleton(loggerFactory); + collection.AddSingleton(typeof(ILogger<>), typeof(Logger<>)); + + var builder = new ContainerBuilder(); + builder.Populate(collection); + + builder.RegisterValidatorConfiguration(validatorConfig); + builder.RegisterEndpointConfiguration(endpointConfig); + builder.RegisterMessageHandlerFactory(messageHandlerFactory); + builder.RegisterEndpoints(endpointConfig); + builder.RegisterSourceRepositories(galleryUrl, indexUrl, galleryDatabase); + builder.RegisterValidators(endpointConfig); + + builder + .RegisterInstance(auditingStorageFactory) + .AsSelf() + .As(); + + builder.RegisterType(); + + var container = builder.Build(); + + return container.Resolve(); + } + + public static PackageValidatorContextEnqueuer CreatePackageValidatorContextEnqueuer( + IStorageQueue queue, + string catalogIndexUrl, + Persistence.IStorageFactory monitoringStorageFactory, + EndpointConfiguration endpointConfig, + ITelemetryService telemetryService, + Func messageHandlerFactory, + ILoggerFactory loggerFactory) + { + if (queue == null) + { + throw new ArgumentNullException(nameof(queue)); + } + + if (string.IsNullOrEmpty(catalogIndexUrl)) + { + throw new ArgumentException(nameof(catalogIndexUrl)); + } + + if (monitoringStorageFactory == null) + { + throw new ArgumentNullException(nameof(monitoringStorageFactory)); + } + + if (telemetryService == null) + { + throw new ArgumentNullException(nameof(telemetryService)); + } + + var collection = new ServiceCollection(); + collection.AddSingleton(loggerFactory); + collection.AddSingleton(typeof(ILogger<>), typeof(Logger<>)); + + var builder = new ContainerBuilder(); + builder.Populate(collection); + + builder.RegisterEndpointConfiguration(endpointConfig); + builder.RegisterEndpoints(endpointConfig); + builder.RegisterMessageHandlerFactory(messageHandlerFactory); + + builder + .RegisterInstance(queue) + .As>(); + + builder + .RegisterInstance(new Uri(catalogIndexUrl)) + .As(); + + builder + .RegisterInstance(telemetryService) + .As(); + + builder + .RegisterType() + .As(); + + builder + .RegisterInstance(GetFront(monitoringStorageFactory)) + .As(); + + builder + .RegisterType() + .As(); + + builder + .RegisterType() + .As(); + + var container = builder.Build(); + + return container.Resolve(); + } + + public static DurableCursor GetFront(Persistence.IStorageFactory storageFactory) + { + var storage = storageFactory.Create(); + return new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue); + } + } +} diff --git a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj index d28f6501e..70bada387 100644 --- a/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj +++ b/src/NuGet.Services.Revalidate/NuGet.Services.Revalidate.csproj @@ -119,7 +119,7 @@ all - 2.74.0 + 2.75.0 diff --git a/src/NuGet.Services.SearchService/App_Start/ApiExceptionFilterAttribute.cs b/src/NuGet.Services.SearchService/App_Start/ApiExceptionFilterAttribute.cs new file mode 100644 index 000000000..a7d75e6bb --- /dev/null +++ b/src/NuGet.Services.SearchService/App_Start/ApiExceptionFilterAttribute.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Net; +using System.Net.Http; +using System.Web.Http.Filters; +using NuGet.Services.AzureSearch; +using NuGet.Services.AzureSearch.SearchService; + +namespace NuGet.Services.SearchService +{ + public class ApiExceptionFilterAttribute : ExceptionFilterAttribute + { + public override void OnException(HttpActionExecutedContext context) + { + switch (context.Exception) + { + case AzureSearchException _: + context.Response = context.Request.CreateResponse( + HttpStatusCode.ServiceUnavailable, + new ErrorResponse("The service is unavailable.")); + break; + + case InvalidSearchRequestException isre: + context.Response = context.Request.CreateResponse( + HttpStatusCode.BadRequest, + new ErrorResponse(isre.Message)); + break; + } + } + + private class ErrorResponse + { + public ErrorResponse(string message) + { + Message = message; + } + + public bool Success { get; } = false; + public string Message { get; } + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.SearchService/App_Start/WebApiConfig.cs b/src/NuGet.Services.SearchService/App_Start/WebApiConfig.cs new file mode 100644 index 000000000..42b43ecdb --- /dev/null +++ b/src/NuGet.Services.SearchService/App_Start/WebApiConfig.cs @@ -0,0 +1,356 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using System.Web.Hosting; +using System.Web.Http; +using System.Web.Http.Cors; +using Autofac; +using Autofac.Extensions.DependencyInjection; +using Autofac.Integration.WebApi; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.DependencyCollector; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.ApplicationInsights.Extensibility.Implementation; +using Microsoft.ApplicationInsights.Extensibility.PerfCounterCollector; +using Microsoft.ApplicationInsights.Extensibility.PerfCounterCollector.QuickPulse; +using Microsoft.ApplicationInsights.Web; +using Microsoft.ApplicationInsights.WindowsServer; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using NuGet.Services.AzureSearch; +using NuGet.Services.AzureSearch.SearchService; +using NuGet.Services.Configuration; +using NuGet.Services.KeyVault; +using NuGet.Services.Logging; +using NuGet.Services.SearchService.Controllers; + +namespace NuGet.Services.SearchService +{ + public static class WebApiConfig + { + private const string ControllerSuffix = "Controller"; + private const string ConfigurationSectionName = "SearchService"; + + public static void Register(HttpConfiguration config) + { + config.Filters.Add(new ApiExceptionFilterAttribute()); + + config.Formatters.Remove(config.Formatters.XmlFormatter); + SetSerializerSettings(config.Formatters.JsonFormatter.SerializerSettings); + + var dependencyResolver = GetDependencyResolver(config); + config.DependencyResolver = dependencyResolver; + + config.EnableCors(new EnableCorsAttribute( + origins: "*", + headers: "Content-Type,If-Match,If-Modified-Since,If-None-Match,If-Unmodified-Since,Accept-Encoding", + methods: "GET,HEAD,OPTIONS", + exposedHeaders: "Content-Type,Content-Length,Last-Modified,Transfer-Encoding,ETag,Date,Vary,Server,X-Hit,X-CorrelationId") + { + PreflightMaxAge = 3600 + }); + + config.MapHttpAttributeRoutes(); + + config.Routes.MapHttpRoute( + name: "Index", + routeTemplate: "", + defaults: new + { + controller = GetControllerName(), + action = nameof(SearchController.IndexAsync), + }); + + config.Routes.MapHttpRoute( + name: "GetStatus", + routeTemplate: "search/diag", + defaults: new + { + controller = GetControllerName(), + action = nameof(SearchController.GetStatusAsync), + }); + + config.Routes.MapHttpRoute( + name: "V2Search", + routeTemplate: "search/query", + defaults: new + { + controller = GetControllerName(), + action = nameof(SearchController.V2SearchAsync), + }); + + config.Routes.MapHttpRoute( + name: "V3Search", + routeTemplate: "query", + defaults: new + { + controller = GetControllerName(), + action = nameof(SearchController.V3SearchAsync), + }); + + config.Routes.MapHttpRoute( + name: "Autocomplete", + routeTemplate: "autocomplete", + defaults: new + { + controller = GetControllerName(), + action = nameof(SearchController.AutocompleteAsync), + }); + + config.EnsureInitialized(); + + HostingEnvironment.QueueBackgroundWorkItem(token => ReloadAuxiliaryFilesAsync(dependencyResolver.Container, token)); + HostingEnvironment.QueueBackgroundWorkItem(token => RefreshSecretsAsync(dependencyResolver.Container, token)); + } + + public static void SetSerializerSettings(JsonSerializerSettings settings) + { + settings.NullValueHandling = NullValueHandling.Ignore; + settings.Converters.Add(new StringEnumConverter()); + } + + private static async Task ReloadAuxiliaryFilesAsync(ILifetimeScope serviceProvider, CancellationToken token) + { + var loader = serviceProvider.Resolve(); + await loader.ReloadContinuouslyAsync(token); + } + + private static async Task RefreshSecretsAsync(ILifetimeScope serviceProvider, CancellationToken token) + { + var loader = serviceProvider.Resolve(); + await loader.RefreshContinuouslyAsync(token); + } + + private static AutofacWebApiDependencyResolver GetDependencyResolver(HttpConfiguration config) + { + var configuration = GetConfiguration(); + + var globalTelemetryDimensions = new Dictionary(); + var applicationInsightsConfiguration = InitializeApplicationInsights(configuration); + + var services = new ServiceCollection(); + services.AddSingleton(configuration.SecretReaderFactory); + services.AddSingleton(applicationInsightsConfiguration.TelemetryConfiguration); + services.AddSingleton(configuration.Root); + services.Add(ServiceDescriptor.Scoped(typeof(IOptionsSnapshot<>), typeof(NonCachingOptionsSnapshot<>))); + services.Configure(configuration.Root.GetSection(ConfigurationSectionName)); + services.Configure(configuration.Root.GetSection(ConfigurationSectionName)); + services.AddAzureSearch(globalTelemetryDimensions); + services.AddSingleton(new TelemetryClient(applicationInsightsConfiguration.TelemetryConfiguration)); + services.AddTransient(); + + var builder = new ContainerBuilder(); + builder.RegisterAssemblyModules(typeof(WebApiConfig).Assembly); + builder.RegisterApiControllers(Assembly.GetExecutingAssembly()); + builder.RegisterWebApiFilterProvider(config); + builder.RegisterWebApiModelBinderProvider(); + builder.Populate(services); + builder.AddAzureSearch(); + + var loggerConfiguration = LoggingSetup.CreateDefaultLoggerConfiguration(withConsoleLogger: false); + var loggerFactory = LoggingSetup.CreateLoggerFactory( + loggerConfiguration, + telemetryConfiguration: applicationInsightsConfiguration.TelemetryConfiguration); + + builder.RegisterInstance(loggerFactory).As(); + builder.RegisterGeneric(typeof(Logger<>)).As(typeof(ILogger<>)); + + var container = builder.Build(); + return new AutofacWebApiDependencyResolver(container); + } + + private static ApplicationInsightsConfiguration InitializeApplicationInsights(RefreshableConfiguration configuration) + { + var instrumentationKey = configuration.Root.GetValue("ApplicationInsights_InstrumentationKey"); + var heartbeatIntervalSeconds = configuration.Root.GetValue("ApplicationInsights_HeartbeatIntervalSeconds", 60); + + var applicationInsightsConfiguration = ApplicationInsights.Initialize( + instrumentationKey, + TimeSpan.FromSeconds(heartbeatIntervalSeconds)); + + applicationInsightsConfiguration.TelemetryConfiguration.TelemetryInitializers.Add(new AzureWebAppTelemetryInitializer()); + applicationInsightsConfiguration.TelemetryConfiguration.TelemetryInitializers.Add(new KnownOperationNameEnricher(new[] + { + GetOperationName(HttpMethod.Get, nameof(SearchController.AutocompleteAsync)), + GetOperationName(HttpMethod.Get, nameof(SearchController.IndexAsync)), + GetOperationName(HttpMethod.Get, nameof(SearchController.GetStatusAsync)), + GetOperationName(HttpMethod.Get, nameof(SearchController.V2SearchAsync)), + GetOperationName(HttpMethod.Get, nameof(SearchController.V3SearchAsync)), + })); + + applicationInsightsConfiguration.TelemetryConfiguration.TelemetryProcessorChainBuilder.Use(next => + { + var processor = new RequestTelemetryProcessor(next); + + processor.SuccessfulResponseCodes.Add(400); + processor.SuccessfulResponseCodes.Add(403); + processor.SuccessfulResponseCodes.Add(404); + processor.SuccessfulResponseCodes.Add(405); + + return processor; + }); + + RegisterApplicationInsightsTelemetryModules(applicationInsightsConfiguration.TelemetryConfiguration); + + return applicationInsightsConfiguration; + } + + private static void RegisterApplicationInsightsTelemetryModules(TelemetryConfiguration configuration) + { + RegisterApplicationInsightsTelemetryModule( + new AppServicesHeartbeatTelemetryModule(), + configuration); + + RegisterApplicationInsightsTelemetryModule( + new AzureInstanceMetadataTelemetryModule(), + configuration); + + RegisterApplicationInsightsTelemetryModule( + new DeveloperModeWithDebuggerAttachedTelemetryModule(), + configuration); + + RegisterApplicationInsightsTelemetryModule( + new UnhandledExceptionTelemetryModule(), + configuration); + + RegisterApplicationInsightsTelemetryModule( + new UnobservedExceptionTelemetryModule(), + configuration); + + var requestTrackingModule = new RequestTrackingTelemetryModule(); + requestTrackingModule.Handlers.Add("Microsoft.VisualStudio.Web.PageInspector.Runtime.Tracing.RequestDataHttpHandler"); + requestTrackingModule.Handlers.Add("System.Web.StaticFileHandler"); + requestTrackingModule.Handlers.Add("System.Web.Handlers.AssemblyResourceLoader"); + requestTrackingModule.Handlers.Add("System.Web.Optimization.BundleHandler"); + requestTrackingModule.Handlers.Add("System.Web.Script.Services.ScriptHandlerFactory"); + requestTrackingModule.Handlers.Add("System.Web.Handlers.TraceHandler"); + requestTrackingModule.Handlers.Add("System.Web.Services.Discovery.DiscoveryRequestHandler"); + requestTrackingModule.Handlers.Add("System.Web.HttpDebugHandler"); + RegisterApplicationInsightsTelemetryModule( + requestTrackingModule, + configuration); + + RegisterApplicationInsightsTelemetryModule( + new ExceptionTrackingTelemetryModule(), + configuration); + + RegisterApplicationInsightsTelemetryModule( + new AspNetDiagnosticTelemetryModule(), + configuration); + + var dependencyTrackingModule = new DependencyTrackingTelemetryModule(); + dependencyTrackingModule.ExcludeComponentCorrelationHttpHeadersOnDomains.Add("core.windows.net"); + dependencyTrackingModule.ExcludeComponentCorrelationHttpHeadersOnDomains.Add("core.chinacloudapi.cn"); + dependencyTrackingModule.ExcludeComponentCorrelationHttpHeadersOnDomains.Add("core.cloudapi.de"); + dependencyTrackingModule.ExcludeComponentCorrelationHttpHeadersOnDomains.Add("core.usgovcloudapi.net"); + dependencyTrackingModule.IncludeDiagnosticSourceActivities.Add("Microsoft.Azure.EventHubs"); + dependencyTrackingModule.IncludeDiagnosticSourceActivities.Add("Microsoft.Azure.ServiceBus"); + RegisterApplicationInsightsTelemetryModule( + dependencyTrackingModule, + configuration); + + RegisterApplicationInsightsTelemetryModule( + new PerformanceCollectorModule(), + configuration); + + RegisterApplicationInsightsTelemetryModule( + new QuickPulseTelemetryModule(), + configuration); + } + + private static void RegisterApplicationInsightsTelemetryModule(ITelemetryModule telemetryModule, TelemetryConfiguration configuration) + { + var existingModule = TelemetryModules.Instance.Modules.SingleOrDefault(m => m.GetType().Equals(telemetryModule.GetType())); + if (existingModule != null) + { + TelemetryModules.Instance.Modules.Remove(existingModule); + } + + telemetryModule.Initialize(configuration); + + TelemetryModules.Instance.Modules.Add(telemetryModule); + } + + private static RefreshableConfiguration GetConfiguration() + { + const string prefix = "APPSETTING_"; + var jsonFile = Path.Combine(HostingEnvironment.MapPath("~/"), @"Settings\local.json"); + + // Load the configuration without injection. This allows us to read KeyVault configuration. + var uninjectedBuilder = new ConfigurationBuilder() + .AddJsonFile(jsonFile) // The JSON file is useful for local development. + .AddEnvironmentVariables(prefix); // Environment variables take precedence. + var uninjectedConfiguration = uninjectedBuilder.Build(); + + // Initialize KeyVault integration. + var secretReaderFactory = new ConfigurationRootSecretReaderFactory(uninjectedConfiguration); + var refreshSecretReaderSettings = new RefreshableSecretReaderSettings(); + var refreshingSecretReaderFactory = new RefreshableSecretReaderFactory(secretReaderFactory, refreshSecretReaderSettings); + var secretReader = refreshingSecretReaderFactory.CreateSecretReader(); + var secretInjector = refreshingSecretReaderFactory.CreateSecretInjector(secretReader); + + // Attempt to inject secrets into all of the configuration strings. + foreach (var pair in uninjectedConfiguration.AsEnumerable()) + { + if (!string.IsNullOrWhiteSpace(pair.Value)) + { + // We can synchronously wait here because we are outside of the request context. It's not great + // but we need to fetch the initial secrets for the cache before activating any controllers or + // asking DI for configuration. + secretInjector.InjectAsync(pair.Value).Wait(); + } + } + + // Reload the configuration with secret injection enabled. This is was is used by the application. + var injectedBuilder = new ConfigurationBuilder() + .AddInjectedJsonFile(jsonFile, secretInjector) + .AddInjectedEnvironmentVariables(prefix, secretInjector); + var injectedConfiguration = injectedBuilder.Build(); + + // Now disable all secrets loads from a non-refresh path. Refresh will be called periodically from a + // background thread. Foreground (request) threads MUST use the cache otherwise there will be a deadlock. + refreshSecretReaderSettings.BlockUncachedReads = true; + + return new RefreshableConfiguration + { + SecretReaderFactory = refreshingSecretReaderFactory, + Root = injectedConfiguration, + }; + } + + private static string GetControllerName() where T : ApiController + { + var typeName = typeof(T).Name; + if (typeName.EndsWith(ControllerSuffix, StringComparison.Ordinal)) + { + return typeName.Substring(0, typeName.Length - ControllerSuffix.Length); + } + + throw new ArgumentException($"The controller type name must end with '{ControllerSuffix}'."); + } + + private static string GetOperationName(HttpMethod verb, string actionName) where T : ApiController + { + return $"{verb} {GetControllerName()}/{actionName}"; + } + + private class RefreshableConfiguration + { + public IRefreshableSecretReaderFactory SecretReaderFactory { get; set; } + public IConfigurationRoot Root { get; set; } + } + } +} \ No newline at end of file diff --git a/src/NuGet.Services.SearchService/ApplicationInsights.config b/src/NuGet.Services.SearchService/ApplicationInsights.config new file mode 100644 index 000000000..6975bcb8c --- /dev/null +++ b/src/NuGet.Services.SearchService/ApplicationInsights.config @@ -0,0 +1,38 @@ + + + + + + + + + + + + search|spider|crawl|Bot|Monitor|AlwaysOn + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/NuGet.Services.SearchService/Controllers/SearchController.cs b/src/NuGet.Services.SearchService/Controllers/SearchController.cs new file mode 100644 index 000000000..c7c12fc02 --- /dev/null +++ b/src/NuGet.Services.SearchService/Controllers/SearchController.cs @@ -0,0 +1,205 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using System.Web.Http; +using System.Web.Http.Description; +using NuGet.Services.AzureSearch.SearchService; +using NuGet.Versioning; + +namespace NuGet.Services.SearchService.Controllers +{ + public class SearchController : ApiController + { + private static readonly NuGetVersion SemVer2Level = new NuGetVersion("2.0.0"); + + private const int DefaultSkip = 0; + private const int DefaultTake = SearchParametersBuilder.DefaultTake; + + private static readonly IReadOnlyDictionary SortBy = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { "relevance", V2SortBy.Popularity }, + { "lastEdited", V2SortBy.LastEditedDesc }, + { "published", V2SortBy.PublishedDesc }, + { "title-asc", V2SortBy.SortableTitleAsc }, + { "title-desc", V2SortBy.SortableTitleDesc }, + { "created-asc", V2SortBy.CreatedAsc }, + { "created-desc", V2SortBy.CreatedDesc }, + { "totalDownloads-asc", V2SortBy.TotalDownloadsAsc }, + { "totalDownloads-desc", V2SortBy.TotalDownloadsDesc }, + }; + + private readonly IAuxiliaryDataCache _auxiliaryDataCache; + private readonly ISearchService _searchService; + private readonly ISearchStatusService _statusService; + + public SearchController( + IAuxiliaryDataCache auxiliaryDataCache, + ISearchService searchService, + ISearchStatusService statusService) + { + _auxiliaryDataCache = auxiliaryDataCache ?? throw new ArgumentNullException(nameof(auxiliaryDataCache)); + _searchService = searchService ?? throw new ArgumentNullException(nameof(searchService)); + _statusService = statusService ?? throw new ArgumentNullException(nameof(statusService)); + } + + [HttpGet] + [ResponseType(typeof(SearchStatusResponse))] + public async Task IndexAsync(HttpRequestMessage request) + { + var result = await GetStatusAsync(SearchStatusOptions.All); + var statusCode = result.Success ? HttpStatusCode.OK : HttpStatusCode.InternalServerError; + + // Hide all information except the success boolean. This is the root page so we can keep it simple. + result = new SearchStatusResponse + { + Success = result.Success, + }; + + return request.CreateResponse(statusCode, result); + } + + [HttpGet] + [ResponseType(typeof(SearchStatusResponse))] + public async Task GetStatusAsync(HttpRequestMessage request) + { + var result = await GetStatusAsync(SearchStatusOptions.All); + var statusCode = result.Success ? HttpStatusCode.OK : HttpStatusCode.InternalServerError; + + return request.CreateResponse(statusCode, result); + } + + [HttpGet] + public async Task V2SearchAsync( + int? skip = DefaultSkip, + int? take = DefaultTake, + bool? ignoreFilter = false, + bool? countOnly = false, + bool? prerelease = false, + string semVerLevel = null, + string q = null, + string sortBy = null, + bool? luceneQuery = true, + string packageType = null, + bool? debug = false) + { + await EnsureInitializedAsync(); + + var request = new V2SearchRequest + { + Skip = skip ?? DefaultSkip, + Take = take ?? DefaultTake, + IgnoreFilter = ignoreFilter ?? false, + CountOnly = countOnly ?? false, + IncludePrerelease = prerelease ?? false, + IncludeSemVer2 = GetIncludeSemVer2(semVerLevel), + Query = q, + SortBy = GetSortBy(sortBy), + LuceneQuery = luceneQuery ?? true, + PackageType = packageType, + ShowDebug = debug ?? false, + }; + + return await _searchService.V2SearchAsync(request); + } + + [HttpGet] + public async Task V3SearchAsync( + int? skip = DefaultSkip, + int? take = DefaultTake, + bool? prerelease = false, + string semVerLevel = null, + string q = null, + string packageType = null, + bool? debug = false) + { + await EnsureInitializedAsync(); + + var request = new V3SearchRequest + { + Skip = skip ?? DefaultSkip, + Take = take ?? DefaultTake, + IncludePrerelease = prerelease ?? false, + IncludeSemVer2 = GetIncludeSemVer2(semVerLevel), + Query = q, + PackageType = packageType, + ShowDebug = debug ?? false, + }; + + return await _searchService.V3SearchAsync(request); + } + + [HttpGet] + public async Task AutocompleteAsync( + int? skip = DefaultSkip, + int? take = DefaultTake, + bool? prerelease = false, + string semVerLevel = null, + string q = null, + string id = null, + string packageType = null, + bool? debug = false) + { + await EnsureInitializedAsync(); + + // If only "id" is provided, find package versions. Otherwise, find package Ids. + var type = (q != null || id == null) + ? AutocompleteRequestType.PackageIds + : AutocompleteRequestType.PackageVersions; + + var request = new AutocompleteRequest + { + Skip = skip ?? DefaultSkip, + Take = take ?? DefaultTake, + IncludePrerelease = prerelease ?? false, + IncludeSemVer2 = GetIncludeSemVer2(semVerLevel), + Query = q ?? id, + Type = type, + PackageType = packageType, + ShowDebug = debug ?? false, + }; + + return await _searchService.AutocompleteAsync(request); + } + + private async Task EnsureInitializedAsync() + { + /// Ensure the auxiliary data is loaded before processing a request. This is necessary because the response + /// builder depends on , which requires that the auxiliary files have + /// been loaded at least once. + await _auxiliaryDataCache.EnsureInitializedAsync(); + } + + private async Task GetStatusAsync(SearchStatusOptions options) + { + var assemblyForMetadata = typeof(SearchController).Assembly; + return await _statusService.GetStatusAsync(options, assemblyForMetadata); + } + + private static V2SortBy GetSortBy(string sortBy) + { + if (sortBy == null || !SortBy.TryGetValue(sortBy, out var parsedSortBy)) + { + parsedSortBy = V2SortBy.Popularity; + } + + return parsedSortBy; + } + + private static bool GetIncludeSemVer2(string semVerLevel) + { + if (!NuGetVersion.TryParse(semVerLevel, out var semVerLevelVersion)) + { + return false; + } + else + { + return semVerLevelVersion >= SemVer2Level; + } + } + } +} diff --git a/src/NuGet.Services.SearchService/Global.asax b/src/NuGet.Services.SearchService/Global.asax new file mode 100644 index 000000000..ebae4c612 --- /dev/null +++ b/src/NuGet.Services.SearchService/Global.asax @@ -0,0 +1 @@ +<%@ Application Codebehind="Global.asax.cs" Inherits="NuGet.Services.SearchService.WebApiApplication" Language="C#" %> diff --git a/src/NuGet.Services.SearchService/Global.asax.cs b/src/NuGet.Services.SearchService/Global.asax.cs new file mode 100644 index 000000000..5970f0692 --- /dev/null +++ b/src/NuGet.Services.SearchService/Global.asax.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Web.Http; + +namespace NuGet.Services.SearchService +{ + public class WebApiApplication : System.Web.HttpApplication + { + protected void Application_Start() + { + GlobalConfiguration.Configure(WebApiConfig.Register); + } + } +} diff --git a/src/NuGet.Services.SearchService/NuGet.Services.SearchService.csproj b/src/NuGet.Services.SearchService/NuGet.Services.SearchService.csproj new file mode 100644 index 000000000..28e1b5d73 --- /dev/null +++ b/src/NuGet.Services.SearchService/NuGet.Services.SearchService.csproj @@ -0,0 +1,141 @@ + + + + + Debug + AnyCPU + + + 2.0 + {DD089AB9-6AB3-4ACA-8D63-C95A7935B2A7} + {349c5851-65df-11da-9384-00065b846f21};{fae04ec0-301f-11d3-bf4b-00c04f79efbc} + Library + Properties + NuGet.Services.SearchService + NuGet.Services.SearchService + v4.7.2 + true + + + + + + + + + + + true + full + false + bin\ + DEBUG;TRACE + prompt + 4 + + + true + pdbonly + true + bin\ + TRACE + prompt + 4 + + + + + + + + + Designer + + + + + + + + Global.asax + + + + + + + Designer + PreserveNewest + + + + + Web.config + + + Web.config + + + + + 4.2.0 + + + 0.3.0 + runtime; build; native; contentfiles; analyzers + all + + + 2.12.0 + + + 2.12.0 + + + 5.2.7 + + + 5.2.7 + + + + + {1a53fe3d-8041-4773-942f-d73aef5b82b2} + NuGet.Services.AzureSearch + + + + 10.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + + + + + + + + + True + True + 22744 + / + http://localhost:21751/ + False + False + + + False + + + + + \ No newline at end of file diff --git a/src/NuGet.Services.SearchService/Properties/AssemblyInfo.cs b/src/NuGet.Services.SearchService/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..608a7ac82 --- /dev/null +++ b/src/NuGet.Services.SearchService/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Services.SearchService")] +[assembly: ComVisible(false)] +[assembly: Guid("dd089ab9-6ab3-4aca-8d63-c95a7935b2a7")] diff --git a/src/NuGet.Services.SearchService/README.md b/src/NuGet.Services.SearchService/README.md new file mode 100644 index 000000000..145f21858 --- /dev/null +++ b/src/NuGet.Services.SearchService/README.md @@ -0,0 +1,367 @@ +## Overview + +**Subsystem: Search 🔎** + +This project contains the search service, the microservice for searching NuGet packages. The search service is an +ASP.NET MVC web application that communicates directly with an existing +[Azure Search](https://azure.microsoft.com/en-us/services/search/) resource in Azure. It can be considered as an adapter +between clients expecting a NuGet-owned protocol and Azure Search, which returns documents with their own Azure Search +schema unrelated to NuGet. + +The primary purpose of the service is to provide metadata about packages most relevant to given customer search text. +However the service has several endpoints meant for a variety of scenarios: both documented REST API contracts as well +as implementation details for [NuGetGallery](https://github.com/NuGet/NuGetGallery) (a.k.a the gallery) search +functionality. + +The officially documented endpoints on the search service are: + +- [`/query`](#query---v3-search-endpoint) - an implementation of the NuGet V3 API [Search resource](https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource) +- [`/autocomplete`](#autocomplete---v3-autocomplete-endpoint) - an implementation of the NuGet V3 API [Autocomplete resource](https://docs.microsoft.com/en-us/nuget/api/search-autocomplete-service-resource) + +Several other endpoints exist as implementation details (i.e. their API surface area is not guaranteed to be stable): + +- [`/search/query`](#searchquery---internal-v2-search-endpoint) - used by the NuGetGallery to fulfill package searches as well some V2 OData queries +- [`/search/diag`](#searchdiag---diagnostic-information-for-monitoring) - used by monitoring for diagnostic information about a running instance of the service +- [`/`](#---health-endpoint-for-load-balancers) - used by infrastructure (such as a load balancer) in front of the service to determine if it is healthy, i.e. a health probe + +The search service can be considered a read-only, egress service. The service expects the configured Azure Search +resource to already be populated with package metadata. The responsibility of the service is to accept user queries, +map the queries to Azure Search REST API calls, and map the resulting Azure Search documents to a JSON shape that the +customer expects. + +The search text is the value passed to the `q` parameter on the `/query` and `/search/query` endpoints. This search +text supports a [basic set of operations](https://docs.microsoft.com/en-us/nuget/consume-packages/finding-and-choosing-packages#search-syntax), +loosely mimicking a small subset of Lucene syntax. + +No authentication is required for accessing any of the endpoints on the service. All endpoints support HTTP GET, receive +any parameters via the query string, and return JSON. + +## Multiple service instances ✅ + +This service is read-only. Therefore, as many instances of the service can be deployed as desired and there will be no +concurrency issues. In fact, nuget.org deploys at least 2 instances of the service to 4 distinct Azure regions. Each +region has its own search service and Azure Search resource (for BCDR reasons). + +The service itself is stateless, depending on external state that is persisted in the configured Azure Search resource. + +## Azure Search indexes + +For more information about the Azure Search indexes that the search service uses, see +[Azure Search indexes](../../docs/Azure-Search-indexes.md). Both the search and hijack index are used. + +## Auxiliary files + +For more information about all files used by the search subsystem, see [search auxiliary files](../../docs/Search-auxiliary-files.md). +A subset of the auxiliary files are used by the search service. The files used are: + + - [`downloads/downloads.v2.json`](../../docs/Search-auxiliary-files.md#download-count-data) - for stitching the latest download count number, per ID and version + - [`verified-packages/verified-packages.v1.json`](../../docs/Search-auxiliary-files.md#verified-packages-data) - for the verified boolean + - [`popularity-transfers/popularity-transfers.v1.json`](../../docs/Search-auxiliary-files.md#popularity-transfer-data) - for monitoring + +## Endpoints + +All endpoints provided the service exist on the [`SearchController`](Controllers/SearchController.cs). + +### `/query` - V3 search endpoint + +This endpoint is used primarily by Visual Studio Package Manager UI. When user's click "Manage NuGet Packages" in +Visual Studio and select the "Browse" tab, search will go directly against the search service. As of .NET 5.0, the +.NET CLI queries the search service via the `dotnet tool search` command to search for packages with package type +`DotnetTool`. On nuget.org, this endpoint is also heavily used by third party applications and scripts. + +As with all V3 resources, the specific URL that the client uses is discovered from the +[service index](https://docs.microsoft.com/en-us/nuget/api/service-index). Additionally, this endpoint supports an +optional `debug=true` parameter which shows the raw Azure Search document and other diagnostic information. + +The parameters and response body are documented in the NuGet V3 API +[Search resource](https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource) reference. + +This endpoint exclusively uses the [search index](../../docs/Azure-Search-indexes.md#search-index). + +### `/autocomplete` - V3 autocomplete endpoint + +This endpoint is used by the Package Manager Console in Visual Studio for package ID table completion for commands like +`Install-Package`. Historically, this endpoint was used by the project.json editor in Visual Studio, however this +scenario is now entirely deprecated. + +As with all V3 resources, the specific URL that the client uses is discovered from the +[service index](https://docs.microsoft.com/en-us/nuget/api/service-index). + +The parameters and response body are documented in the NuGet V3 API +[Autocomplete resource](https://docs.microsoft.com/en-us/nuget/api/search-autocomplete-service-resource) reference. +Additionally, this endpoint supports an optional `debug=true` parameter which shows the raw Azure Search document and +other diagnostic information. + +This endpoint exclusively uses the [search index](../../docs/Azure-Search-indexes.md#search-index) to enumerate both +package IDs and package versions. + +### `/search/query` - internal V2 search endpoint + +This endpoint is used by NuGetGallery in several scenarios when the gallery is configured with an external search +service. Therefore, the contract should be considered unstable by external clients and may change freely to match the +requirements of NuGetGallery. We strongly urge external applications (even NuGet client) to avoid using this endpoint +since it should be considered and implementation detail of NuGetGallery. External clients should use the document V3 +Search endpoint and discover the URL via the service index. + +When the NuGetGallery code "hijacks" an OData query to search service instead of going to the SQL database, this +endpoint is used. Here is a non-exhaustive list of how NuGetGallery endpoints call into search service: + +- V2 API, search + - Gallery URL: `/api/v2/Search()?q=%27json%27` + - Search URL: `/search/query?q=json&skip=0&take=100&sortBy=relevance&luceneQuery=false` + - This is used for package search scenarios much like [V3 search](#query---v3-search-endpoint) when the client is using a V2 + source URL pointing to NuGetGallery. +- V2 API, get metadata of all versions of an ID + - Gallery URL: `/api/v2/FindPackagesById()?id=%27Newtonsoft.Json%27` + - Search URL: `/search/query?q=Id%3A%22Newtonsoft.Json%22&skip=0&take=100&sortBy=created-asc&prerelease=true&ignoreFilter=true` + - This is used for package restore scenarios much like the V3 endpoint to [enumerate package versions](https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#enumerate-package-versions) + but when the client is using ta V2 source URL pointing to NuGetGallery. +- V2 API, get metadata about a specific version + - Gallery URL: `/api/v2/Packages(Id='Newtonsoft.Json',Version='9.0.1')` + - Search URL: `/search/query?q=Id%3A%22Newtonsoft.Json%22+AND+Version%3A%229.0.1%22&skip=0&take=1&sortBy=created-asc&semVerLevel=2.0.0&prerelease=true&ignoreFilter=true` +- Gallery UI, search + - Gallery URL: `/packages?q=json` + - Search URL: `/search/query?q=json&skip=0&take=20&sortBy=relevance&semVerLevel=2.0.0&prerelease=true&luceneQuery=false` + +This endpoint uses both the [search index](../../docs/Azure-Search-indexes.md#search-index) and the [hijack index](../../docs/Azure-Search-indexes.md#hijack-index), +depending on the `ignoreFilter` parameter. + +#### Request parameters + +Name | Type | Notes +------------ | ------- | ----- +q | string | The search terms used to filter packages +skip | integer | The number of results to skip, for pagination +take | integer | The number of results to return, for pagination +prerelease | boolean | `true` or `false` determining whether to include prerelease packages +semVerLevel | string | A SemVer 1.0.0 version string: `1.0.0` or `2.0.0` +ignoreFilter | boolean | `true` to include unlisted packages and ignore the `prerelease` parameter +countOnly | boolean | `true` to return only the total count and no metadata +sortBy | string | Sort results using a specified ordering +packageType | string | Filter results to those with the specified package type +luceneQuery | bool | `true` to treat a `q` starting with `id:` like `packageid:` (yes, it's silly, see [#7366](https://github.com/NuGet/NuGetGallery/issues/7366)) +debug | bool | `true` to shows the raw Azure Search document and other diagnostic information + +If no `q` is provided, all packages should be returned, within the boundaries imposed by skip and take. + +The `skip` parameter defaults to 0. The maximum value is 10000. + +The `take` parameter should be an integer greater than zero. The default value is 20. The maximum value is 1000. + +If `prerelease` is not provided, prerelease packages are excluded. + +The `semVerLevel` query parameter is used to opt-in to +[SemVer 2.0.0 packages](https://github.com/NuGet/Home/wiki/SemVer2-support-for-nuget.org-%28server-side%29#identifying-semver-v200-packages). +If this query parameter is excluded, only packages with SemVer 1.0.0 compatible versions will be returned (with the +[standard NuGet versioning](https://docs.microsoft.com/en-us/nuget/concepts/package-versioning) caveats, such as version strings with 4 integer pieces). +If `semVerLevel=2.0.0` is provided, both SemVer 1.0.0 and SemVer 2.0.0 compatible packages will be returned. See the +[SemVer 2.0.0 support for nuget.org](https://github.com/NuGet/Home/wiki/SemVer2-support-for-nuget.org-%28server-side%29) +for more information. + +The `ignoreFilter` parameter is used to toggle between the search index and the hijack index. Standard package search +or discovery scenarios by keyword will use `ignoreFilter=false`. Metadata look-up scenarios (mainly for NuGet restore) +will use `ignoreFilter=true` which allows the metadata of non-latest and unlisted packages to be seen. The `semVerLevel` +parameter still applies when `ignoreFilter=true` (i.e. not all filters are ignored). + +The `sortBy` parameter supports the following options: + +- `relevance` - sort by relevance, most relevant at the top +- `lastEdited` - sort by last edited timestamp, descending chronological order +- `published` - sort by published timestamp, descending chronological order +- `title-asc` - sort by title, ascending case-insensitive lexicographical order +- `title-desc` - sort by title, descending case-insensitive lexicographical order +- `created-asc` - sort by created timestamp, ascending chronological order +- `created-desc` - sort by created timestamp, descending chronological order +- `totalDownloads-asc` - sort by total downloads count, ascending numerical order +- `totalDownloads-desc` - sort by total downloads count, descending numerical order + +The `relevance` value is used by default or if the provided value is not supported. + +For `title-asc` and `title-desc`, a package's ID is used if the package has no explicit title value. Given that package +`title` is no longer prominently shown in NuGet experiences, this sorting order is only maintained for legacy reasons. + +The `packageType` parameter is a free form input. It defaults to an empty string, which means no filter. If there are no packages with the specified packageType in a request, an empty `data` array will be returned. + +#### Response + +The response is different than the V3 Search response but shares many of the same fields. The fields have slightly +different names (PascalCase instead of camelCase) and are arranged differently. Some of the main differences are: + +- `/search/query` has the following fields that `/query` does not have: + - `Owners` - not present when querying the hijack index + - `Version` - the verbatim/original version found in the .nuspec + - `Copyright` + - `Language` + - `ReleaseNotes` + - `IsLatest` and `IsLatestStable` - respects the the `semVerLevel` parameter, only interesting when `ignoreFilter=true` + - `Listed` - only interesting when `ignoreFilter=true` + - `Created` - created timestamp + - `Published` - published timestamp + - `LastEdited` - last edited timestamp + - `FlattenedDependencies` - the package dependencies structured data but as a flat string using a custom encoding + - `MinClientVersion` + - `Hash` - base64 encoded + - `HashAlgorithm` + - `PackageFileSize` + - `RequiresLicenseAcceptance` +- `/query` has the following fields that `/search/query` does not have: + - `packageTypes` - array of package type objects + - `versions` - full list of versions for that package ID + +The `LastUpdated` property has the same value as `Published` for legacy reasons. + +The `Dependencies` and `SupportedFrameworks` fields are always empty arrays because NuGetGallery does not used these +values but expects the properties to be present. + +#### Sample response + +```json +{ + "totalHits": 1, + "data": [ + { + "PackageRegistration": { + "Id": "BaseTestPackage.Unlisted", + "DownloadCount": 93, + "Verified": false, + "Owners": [], + "PopularityTransfers": [] + }, + "Version": "1.1.0", + "NormalizedVersion": "1.1.0", + "Title": "BaseTestPackage.Unlisted", + "Description": "A package for testing unlisted status.", + "Summary": "", + "Authors": "jver", + "Tags": "", + "IsLatestStable": false, + "IsLatest": false, + "Listed": false, + "Created": "2019-07-22T15:48:32.107+00:00", + "Published": "1900-01-01T00:00:00+00:00", + "LastUpdated": "1900-01-01T00:00:00+00:00", + "LastEdited": "2019-07-22T15:52:43.053+00:00", + "DownloadCount": 93, + "FlattenedDependencies": "", + "Dependencies": [], + "SupportedFrameworks": [], + "Hash": "kRvVPTmvFRa+EaKmYJhitnHbZLexclm3fLtKJGwigbExRlrmOCtYH+zXfGSeuxCE980x3aSgqwM9V5PaNlnFRw==", + "HashAlgorithm": "SHA512", + "PackageFileSize": 9988, + "RequiresLicenseAcceptance": false + } + ] +} +``` + +### `/search/diag` - diagnostic information for monitoring + +This endpoint is used to show diagnostic information. This enables the following monitoring scenarios: + +- recent secret reload from KeyVault, via the `Server.LastServiceRefreshTime` property +- latest catalog data is in the search index, via the `SearchIndex.LastCommitTimestamp` property +- latest catalog data is in the hijack index, via the `HijackIndex.LastCommitTimestamp` property +- recent auxiliary file reload, via the `AuxiliaryFiles.Loaded` property + +The other properties are just helpful for live site investigations. + +The contract of the response body is unstable and can change freely over time given the internal monitoring systems +react to the changes appropriately. External client software should not use this endpoint. + +An HTTP `200 OK` is returned if the minimum dependencies are available for the search service. If there is a problem, +HTTP `500 Internal Server Error` is returned. + + +### `/` - health endpoint for load balancers + +The endpoint internally fetches the same data as `/search/diag` but only returns a simple success boolean. + +An HTTP `200 OK` is returned if the minimum dependencies are available for the search service. If there is a problem, +HTTP `500 Internal Server Error` is returned. + +## Running the service + +Uses one of the following approaches to modify the `Settings/local.json` file with configuration +values. Once this configuration files has the settings you'd like, you can launch the service in Visual Studio using +F5. This will start the service in IIS Express and open your web browser to the running service. + +### Using personal Azure resources + +To use you own resources, you need to initialize the indexes in an Azure Search resource and the auxiliary files in +Azure Blob Storage. The easiest way to do this is using the [Db2AzureSearch tool](../NuGet.Jobs.Db2AzureSearch/README.md). +This tool populates the search and hijack indexes in the configured Azure Search resource and initialize the initial +auxiliary data files. After the tool finishes, simply configure the Search Service to point to the same Azure Search and +Blob Storage container. + +You don't necessarily need to run Catalog2AzureSearch or Auxiliary2AzureSearch since these two jobs keep the indexes and +auxiliary files up to data after Db2AzureSearch has already initialized the data. For testing, you can typically get by +with static data that isn't staying up to data all the time. + +The `PLACEHOLDER` values need to match whatever you used when running Db2AzureSearch, except for `SearchServiceApiKey` +which can be a read-only (query) key instead of the admin key used by Db2AzureSearch. + +The `ApplicationInsights_InstrumentationKey` setting is optional and can be removed. + +You can use different values for `FlatContainerBaseUrl`, `FlatContainerContainerName`, `SemVer1RegistrationsBaseUrl`, +and `SemVer2RegistrationsBaseUrl` but the impact is low. These settings are just used for building URLs returned in the +service responses and are not called into at runtime. + +```json +{ + "ApplicationInsights_InstrumentationKey": "PLACEHOLDER", + "SearchService": { + "AllIconsInFlatContainer": true, + "FlatContainerBaseUrl": "https://apidev.nugettest.org/", + "FlatContainerContainerName": "v3-flatcontainer", + "HijackIndexName": "PLACEHOLDER", + "SearchIndexName": "PLACEHOLDER", + "SearchServiceApiKey": "PLACEHOLDER", + "SearchServiceName": "PLACEHOLDER", + "SemVer1RegistrationsBaseUrl": "https://apidev.nugettest.org/v3/registration5-semver1/", + "SemVer2RegistrationsBaseUrl": "https://apidev.nugettest.org/v3/registration5-gz-semver2/", + "StorageConnectionString": "PLACEHOLDER", + "StorageContainer": "PLACEHOLDER" + } +} +``` + +### Using DEV resources + +The easiest way to run the job if you are on the nuget.org server team is to use the DEV environment resources. This can +be done by pointing configuration to a DEV Azure Search region and installing the certificate used to authenticate as +our client AAD app registration into your `CurrentUser` certificate store. You can look up the `PLACEHOLDER` values in +the internal deployment/configuration repository. The `XXX` is the current index number used in DEV. + +The `ApplicationInsights_InstrumentationKey` setting is optional and can be removed. + +```json +{ + "ApplicationInsights_InstrumentationKey": "PLACEHOLDER", + "KeyVault_ClientId": "PLACEHOLDER", + "KeyVault_CertificateThumbprint": "PLACEHOLDER", + "KeyVault_ValidateCertificate": true, + "KeyVault_StoreName": "My", + "KeyVault_StoreLocation": "CurrentUser", + "KeyVault_VaultName": "PLACEHOLDER", + "SearchService": { + "AllIconsInFlatContainer": true, + "FlatContainerBaseUrl": "https://apidev.nugettest.org/", + "FlatContainerContainerName": "v3-flatcontainer", + "HijackIndexName": "hijack-XXX", + "SearchIndexName": "search-XXX", + "SearchServiceApiKey": "PLACEHOLDER", + "SearchServiceName": "PLACEHOLDER", + "SemVer1RegistrationsBaseUrl": "https://apidev.nugettest.org/v3/registration5-semver1/", + "SemVer2RegistrationsBaseUrl": "https://apidev.nugettest.org/v3/registration5-gz-semver2/", + "StorageConnectionString": "PLACEHOLDER", + "StorageContainer": "v3-azuresearch-XXX" + } +} +``` + +## Prerequisites + +- **Azure Search**. The search and hijack indexes must already be initialized in the Azure Search resource. +- **Azure Blob Storage**. Several auxiliary files are read from Blob Storage and reloaded periodically. + +For more information, please refer to the [`Db2AzureSearch`](../NuGet.Jobs.Db2AzureSearch/README.md) tool. diff --git a/src/NuGet.Services.SearchService/Settings/local.json b/src/NuGet.Services.SearchService/Settings/local.json new file mode 100644 index 000000000..7a73a41bf --- /dev/null +++ b/src/NuGet.Services.SearchService/Settings/local.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/src/NuGet.Services.SearchService/Web.Debug.config b/src/NuGet.Services.SearchService/Web.Debug.config new file mode 100644 index 000000000..72fae74a8 --- /dev/null +++ b/src/NuGet.Services.SearchService/Web.Debug.config @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/src/NuGet.Services.SearchService/Web.Release.config b/src/NuGet.Services.SearchService/Web.Release.config new file mode 100644 index 000000000..817ae52c3 --- /dev/null +++ b/src/NuGet.Services.SearchService/Web.Release.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/NuGet.Services.SearchService/Web.config b/src/NuGet.Services.SearchService/Web.config new file mode 100644 index 000000000..6f3cd7545 --- /dev/null +++ b/src/NuGet.Services.SearchService/Web.config @@ -0,0 +1,84 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/NuGet.Services.V3/CommitCollectorConfiguration.cs b/src/NuGet.Services.V3/CommitCollectorConfiguration.cs new file mode 100644 index 000000000..156eaee85 --- /dev/null +++ b/src/NuGet.Services.V3/CommitCollectorConfiguration.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.V3 +{ + public class CommitCollectorConfiguration : ICommitCollectorConfiguration + { + public int MaxConcurrentCatalogLeafDownloads { get; set; } = 64; + public string Source { get; set; } + public TimeSpan HttpClientTimeout { get; set; } = TimeSpan.FromMinutes(10); + } +} diff --git a/src/NuGet.Services.V3/CommitCollectorHost.cs b/src/NuGet.Services.V3/CommitCollectorHost.cs new file mode 100644 index 000000000..db11739a3 --- /dev/null +++ b/src/NuGet.Services.V3/CommitCollectorHost.cs @@ -0,0 +1,55 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Newtonsoft.Json.Linq; +using NuGet.Services.Metadata.Catalog; + +namespace NuGet.Services.V3 +{ + /// + /// This is a minimal integration class between the core of the collectors based on NuGet.Jobs infrastructure and + /// the overly complex collector infrastructure that we have today. + /// + public class CommitCollectorHost : CommitCollector, ICollector + { + private readonly ICommitCollectorLogic _logic; + + public CommitCollectorHost( + ICommitCollectorLogic logic, + ITelemetryService telemetryService, + Func handlerFunc, + IOptionsSnapshot options) : base( + new Uri(options.Value.Source), + telemetryService, + handlerFunc, + options.Value.HttpClientTimeout) + { + _logic = logic ?? throw new ArgumentNullException(nameof(logic)); + } + + protected override async Task> CreateBatchesAsync( + IEnumerable catalogItems) + { + return await _logic.CreateBatchesAsync(catalogItems); + } + + protected override async Task OnProcessBatchAsync( + CollectorHttpClient client, + IEnumerable items, + JToken context, + DateTime commitTimeStamp, + bool isLastBatch, + CancellationToken cancellationToken) + { + await _logic.OnProcessBatchAsync(items); + + return true; + } + } +} diff --git a/src/NuGet.Services.V3/CommitCollectorUtility.cs b/src/NuGet.Services.V3/CommitCollectorUtility.cs new file mode 100644 index 000000000..eb62f7de4 --- /dev/null +++ b/src/NuGet.Services.V3/CommitCollectorUtility.cs @@ -0,0 +1,169 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Protocol.Catalog; +using NuGet.Services.Metadata.Catalog; + +namespace NuGet.Services.V3 +{ + public class CommitCollectorUtility + { + private readonly ICatalogClient _catalogClient; + private readonly IV3TelemetryService _telemetryService; + private readonly IOptionsSnapshot _options; + private readonly ILogger _logger; + + public CommitCollectorUtility( + ICatalogClient catalogClient, + IV3TelemetryService telemetryService, + IOptionsSnapshot options, + ILogger logger) + { + _catalogClient = catalogClient ?? throw new ArgumentNullException(nameof(catalogClient)); + _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (_options.Value.MaxConcurrentCatalogLeafDownloads <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(options), + $"The {nameof(ICommitCollectorConfiguration.MaxConcurrentCatalogLeafDownloads)} must be greater than zero."); + } + } + + public IEnumerable CreateSingleBatch(IEnumerable catalogItems) + { + if (!catalogItems.Any()) + { + return Enumerable.Empty(); + } + + var maxCommitTimestamp = catalogItems.Max(x => x.CommitTimeStamp); + + return new[] + { + new CatalogCommitItemBatch( + catalogItems, + key: null, + commitTimestamp: maxCommitTimestamp), + }; + } + + public List GetLatestPerIdentity(IEnumerable items) + { + return items + .GroupBy(x => x.PackageIdentity) + .Select(GetLatestPerSingleIdentity) + .ToList(); + } + + private CatalogCommitItem GetLatestPerSingleIdentity(IEnumerable entries) + { + CatalogCommitItem max = null; + foreach (var entry in entries) + { + if (max == null) + { + max = entry; + continue; + } + + if (!StringComparer.OrdinalIgnoreCase.Equals(max.PackageIdentity, entry.PackageIdentity)) + { + throw new InvalidOperationException("The entries compared should have the same package identity."); + } + + if (entry.CommitTimeStamp > max.CommitTimeStamp) + { + max = entry; + } + else if (entry.CommitTimeStamp == max.CommitTimeStamp) + { + const string message = "There are multiple catalog leaves for a single package at one time."; + _logger.LogError( + message + " ID: {PackageId}, version: {PackageVersion}, commit timestamp: {CommitTimestamp:O}", + entry.PackageIdentity.Id, + entry.PackageIdentity.Version.ToFullString(), + entry.CommitTimeStamp); + throw new InvalidOperationException(message); + } + } + + return max; + } + + public ConcurrentBag>> GroupById(List latestItems) + { + var workEnumerable = latestItems + .GroupBy(x => x.PackageIdentity.Id, StringComparer.OrdinalIgnoreCase) + .Select(x => new IdAndValue>(x.Key, x.ToList())); + + var allWork = new ConcurrentBag>>(workEnumerable); + return allWork; + } + + public async Task> GetEntryToDetailsLeafAsync( + IEnumerable entries) + { + var packageDetailsEntries = entries.Where(IsOnlyPackageDetails); + var allWork = new ConcurrentBag(packageDetailsEntries); + var output = new ConcurrentBag>(); + + using (_telemetryService.TrackCatalogLeafDownloadBatch(allWork.Count)) + { + var tasks = Enumerable + .Range(0, _options.Value.MaxConcurrentCatalogLeafDownloads) + .Select(async x => + { + await Task.Yield(); + while (allWork.TryTake(out var work)) + { + try + { + _logger.LogInformation( + "Downloading catalog leaf for {PackageId} {Version}: {Url}", + work.PackageIdentity.Id, + work.PackageIdentity.Version.ToNormalizedString(), + work.Uri.AbsoluteUri); + + var leaf = await _catalogClient.GetPackageDetailsLeafAsync(work.Uri.AbsoluteUri); + output.Add(KeyValuePair.Create(work, leaf)); + } + catch (Exception ex) + { + _logger.LogError( + ex, + "An exception was thrown when fetching the package details leaf for {Id} {Version}. " + + "The URL is {Url}", + work.PackageIdentity.Id, + work.PackageIdentity.Version, + work.Uri.AbsoluteUri); + throw; + } + } + }) + .ToList(); + + await Task.WhenAll(tasks); + + return output.ToDictionary( + x => x.Key, + x => x.Value, + ReferenceEqualityComparer.Default); + } + } + + public static bool IsOnlyPackageDetails(CatalogCommitItem e) + { + return e.IsPackageDetails && !e.IsPackageDelete; + } + } +} diff --git a/src/NuGet.Services.V3/DefaultBlobRequestOptions.cs b/src/NuGet.Services.V3/DefaultBlobRequestOptions.cs new file mode 100644 index 000000000..aabcdbbc7 --- /dev/null +++ b/src/NuGet.Services.V3/DefaultBlobRequestOptions.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.WindowsAzure.Storage.Blob; +using Microsoft.WindowsAzure.Storage.RetryPolicies; + +namespace NuGet.Services.V3 +{ + public static class DefaultBlobRequestOptions + { + public static BlobRequestOptions Create() + { + return new BlobRequestOptions + { + ServerTimeout = TimeSpan.FromMinutes(2), + MaximumExecutionTime = TimeSpan.FromMinutes(10), + LocationMode = LocationMode.PrimaryThenSecondary, + RetryPolicy = new ExponentialRetry(), + }; + } + } +} diff --git a/src/NuGet.Services.V3/DependencyInjectionExtensions.cs b/src/NuGet.Services.V3/DependencyInjectionExtensions.cs new file mode 100644 index 000000000..13c95122b --- /dev/null +++ b/src/NuGet.Services.V3/DependencyInjectionExtensions.cs @@ -0,0 +1,122 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using Autofac; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage.Blob; +using Microsoft.WindowsAzure.Storage.RetryPolicies; +using NuGet.Jobs.Validation; +using NuGet.Protocol.Catalog; +using NuGet.Protocol.Registration; +using NuGet.Services.FeatureFlags; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog; +using NuGetGallery; +using NuGetGallery.Diagnostics; +using NuGetGallery.Features; + +namespace NuGet.Services.V3 +{ + public static class DependencyInjectionExtensions + { + private const string FeatureFlagBindingKey = nameof(FeatureFlagBindingKey); + + private static readonly TimeSpan FeatureFlagServerTimeout = TimeSpan.FromSeconds(30); + private static readonly TimeSpan FeatureFlagMaxExecutionTime = TimeSpan.FromMinutes(10); + + public static IServiceCollection AddV3(this IServiceCollection services, IDictionary telemetryGlobalDimensions) + { + services + .AddTransient(p => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate + }); + + services + .AddTransient(p => (HttpMessageHandler)new TelemetryHandler( + p.GetRequiredService(), + p.GetRequiredService())); + + services.AddSingleton(p => new HttpClient(p.GetRequiredService())); + + services + .AddTransient(p => new CatalogClient( + p.GetRequiredService(), + p.GetRequiredService>())); + + services.AddTransient(); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + services.AddTransient(p => new TelemetryService( + p.GetRequiredService(), + telemetryGlobalDimensions)); + services.AddTransient(); + + return services; + } + + public static void AddFeatureFlags(this ContainerBuilder containerBuilder) + { + containerBuilder + .Register(c => + { + var options = c.Resolve>(); + var requestOptions = new BlobRequestOptions + { + ServerTimeout = FeatureFlagServerTimeout, + MaximumExecutionTime = FeatureFlagMaxExecutionTime, + LocationMode = LocationMode.PrimaryThenSecondary, + RetryPolicy = new ExponentialRetry(), + }; + + return new CloudBlobClientWrapper( + options.Value.ConnectionString, + requestOptions); + }) + .Keyed(FeatureFlagBindingKey); + + containerBuilder + .Register(c => new CloudBlobCoreFileStorageService( + c.ResolveKeyed(FeatureFlagBindingKey), + c.Resolve(), + c.Resolve())) + .Keyed(FeatureFlagBindingKey); + + containerBuilder + .Register(c => new FeatureFlagFileStorageService( + c.ResolveKeyed(FeatureFlagBindingKey))) + .As(); + } + + public static IServiceCollection AddFeatureFlags(this IServiceCollection services) + { + services + .AddTransient(p => + { + var options = p.GetRequiredService>(); + return new FeatureFlagOptions + { + RefreshInterval = options.Value.RefreshInternal, + }; + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.AddSingleton(); + services.AddSingleton(); + + return services; + } + } +} diff --git a/src/NuGet.Services.V3/ICollector.cs b/src/NuGet.Services.V3/ICollector.cs new file mode 100644 index 000000000..7955dfd2a --- /dev/null +++ b/src/NuGet.Services.V3/ICollector.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog; + +namespace NuGet.Services.V3 +{ + public interface ICollector + { + Task RunAsync(ReadWriteCursor front, ReadCursor back, CancellationToken cancellationToken); + } +} diff --git a/src/NuGet.Services.V3/ICommitCollectorConfiguration.cs b/src/NuGet.Services.V3/ICommitCollectorConfiguration.cs new file mode 100644 index 000000000..026ec37cf --- /dev/null +++ b/src/NuGet.Services.V3/ICommitCollectorConfiguration.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.V3 +{ + public interface ICommitCollectorConfiguration + { + int MaxConcurrentCatalogLeafDownloads { get; } + string Source { get; } + TimeSpan HttpClientTimeout { get; } + } +} diff --git a/src/NuGet.Services.V3/ICommitCollectorLogic.cs b/src/NuGet.Services.V3/ICommitCollectorLogic.cs new file mode 100644 index 000000000..3476a25da --- /dev/null +++ b/src/NuGet.Services.V3/ICommitCollectorLogic.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog; + +namespace NuGet.Services.V3 +{ + public interface ICommitCollectorLogic + { + Task> CreateBatchesAsync(IEnumerable catalogItems); + Task OnProcessBatchAsync(IEnumerable items); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.V3/IV3TelemetryService.cs b/src/NuGet.Services.V3/IV3TelemetryService.cs new file mode 100644 index 000000000..a781b4291 --- /dev/null +++ b/src/NuGet.Services.V3/IV3TelemetryService.cs @@ -0,0 +1,12 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services.V3 +{ + public interface IV3TelemetryService + { + IDisposable TrackCatalogLeafDownloadBatch(int count); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.V3/NuGet.Services.V3.csproj b/src/NuGet.Services.V3/NuGet.Services.V3.csproj new file mode 100644 index 000000000..369570984 --- /dev/null +++ b/src/NuGet.Services.V3/NuGet.Services.V3.csproj @@ -0,0 +1,99 @@ + + + + + + Debug + AnyCPU + {C3F9A738-9759-4B2B-A50D-6507B28A659B} + Library + Properties + NuGet.Services.V3 + NuGet.Services.V3 + v4.7.2 + 512 + true + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0.3.0 + runtime; build; native; contentfiles; analyzers + all + + + + + {e97f23b8-ecb0-4afa-b00c-015c39395fef} + NuGet.Services.Metadata.Catalog + + + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} + NuGet.Protocol.Catalog + + + {fa87d075-a934-4443-8d0b-5db32640b6d7} + Validation.Common.Job + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + + \ No newline at end of file diff --git a/src/NuGet.Services.V3/Properties/AssemblyInfo.cs b/src/NuGet.Services.V3/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..0c48a92ed --- /dev/null +++ b/src/NuGet.Services.V3/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Services.V3")] +[assembly: ComVisible(false)] +[assembly: Guid("c3f9a738-9759-4b2b-a50d-6507b28a659b")] diff --git a/src/NuGet.Services.V3/Registration/IRegistrationClient.cs b/src/NuGet.Services.V3/Registration/IRegistrationClient.cs new file mode 100644 index 000000000..43aa84920 --- /dev/null +++ b/src/NuGet.Services.V3/Registration/IRegistrationClient.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; + +namespace NuGet.Protocol.Registration +{ + public interface IRegistrationClient + { + Task GetIndexOrNullAsync(string indexUrl); + Task GetLeafAsync(string leafUrl); + Task GetPageAsync(string pageUrl); + } +} \ No newline at end of file diff --git a/src/NuGet.Services.V3/Registration/Models/ICommitted.cs b/src/NuGet.Services.V3/Registration/Models/ICommitted.cs new file mode 100644 index 000000000..59de829b8 --- /dev/null +++ b/src/NuGet.Services.V3/Registration/Models/ICommitted.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Protocol.Registration +{ + public interface ICommitted + { + string CommitId { get; set; } + DateTimeOffset CommitTimestamp { get; set; } + } +} diff --git a/src/NuGet.Services.V3/Registration/Models/RegistrationCatalogEntry.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationCatalogEntry.cs new file mode 100644 index 000000000..6a23cf1fa --- /dev/null +++ b/src/NuGet.Services.V3/Registration/Models/RegistrationCatalogEntry.cs @@ -0,0 +1,82 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; +using NuGet.Protocol.Catalog; + +namespace NuGet.Protocol.Registration +{ + /// + /// Source: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#catalog-entry + /// + public class RegistrationCatalogEntry + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("@type")] + public string Type { get; set; } + + [JsonProperty("authors")] + public string Authors { get; set; } + + [JsonProperty("dependencyGroups")] + public List DependencyGroups { get; set; } + + [JsonProperty("deprecation")] + public PackageDeprecation Deprecation { get; set; } + + [JsonProperty("description")] + public string Description { get; set; } + + [JsonProperty("iconUrl")] + public string IconUrl { get; set; } + + [JsonProperty("id")] + public string PackageId { get; set; } + + [JsonProperty("language")] + public string Language { get; set; } + + [JsonProperty("licenseExpression")] + public string LicenseExpression { get; set; } + + [JsonProperty("licenseUrl")] + public string LicenseUrl { get; set; } + + [JsonProperty("listed")] + public bool Listed { get; set; } + + [JsonProperty("minClientVersion")] + public string MinClientVersion { get; set; } + + [JsonProperty("packageContent")] + public string PackageContent { get; set; } + + [JsonProperty("projectUrl")] + public string ProjectUrl { get; set; } + + [JsonProperty("published")] + public DateTimeOffset? Published { get; set; } + + [JsonProperty("requireLicenseAcceptance")] + public bool RequireLicenseAcceptance { get; set; } + + [JsonProperty("summary")] + public string Summary { get; set; } + + [JsonProperty("tags")] + public List Tags { get; set; } + + [JsonProperty("title")] + public string Title { get; set; } + + [JsonProperty("version")] + public string Version { get; set; } + + [JsonProperty("vulnerabilities")] + public List Vulnerabilities { get; set; } + } +} diff --git a/src/NuGet.Services.V3/Registration/Models/RegistrationContainerContext.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationContainerContext.cs new file mode 100644 index 000000000..848471c9b --- /dev/null +++ b/src/NuGet.Services.V3/Registration/Models/RegistrationContainerContext.cs @@ -0,0 +1,62 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NuGet.Protocol.Catalog; + +namespace NuGet.Protocol.Registration +{ + public class RegistrationContainerContext + { + [JsonProperty("@vocab")] + public string Vocab { get; set; } + + [JsonProperty("catalog")] + public string Catalog { get; set; } + + [JsonProperty("xsd")] + public string Xsd { get; set; } + + [JsonProperty("items")] + public ContextTypeDescription Items { get; set; } + + [JsonProperty("commitTimeStamp")] + public ContextTypeDescription CommitTimestamp { get; set; } + + [JsonProperty("commitId")] + public ContextTypeDescription CommitId { get; set; } + + [JsonProperty("count")] + public ContextTypeDescription Count { get; set; } + + [JsonProperty("parent")] + public ContextTypeDescription Parent { get; set; } + + [JsonProperty("tags")] + public ContextTypeDescription Tags { get; set; } + + [JsonProperty("reasons")] + public ContextTypeDescription Reasons { get; set; } + + [JsonProperty("packageTargetFrameworks")] + public ContextTypeDescription PackageTargetFrameworks { get; set; } + + [JsonProperty("dependencyGroups")] + public ContextTypeDescription DependencyGroups { get; set; } + + [JsonProperty("dependencies")] + public ContextTypeDescription Dependencies { get; set; } + + [JsonProperty("packageContent")] + public ContextTypeDescription PackageContent { get; set; } + + [JsonProperty("published")] + public ContextTypeDescription Published { get; set; } + + [JsonProperty("registration")] + public ContextTypeDescription Registration { get; set; } + + [JsonProperty("vulnerabilities")] + public ContextTypeDescription Vulnerabilities { get; set; } + } +} diff --git a/src/NuGet.Services.V3/Registration/Models/RegistrationIndex.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationIndex.cs new file mode 100644 index 000000000..177d5847d --- /dev/null +++ b/src/NuGet.Services.V3/Registration/Models/RegistrationIndex.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Registration +{ + /// + /// Source: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-index + /// + public class RegistrationIndex : ICommitted + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("@type")] + public List Types { get; set; } + + [JsonProperty("commitId")] + public string CommitId { get; set; } + + [JsonProperty("commitTimeStamp")] + public DateTimeOffset CommitTimestamp { get; set; } + + [JsonProperty("count")] + public int Count { get; set; } + + [JsonProperty("items")] + public List Items { get; set; } + + [JsonProperty("@context")] + public RegistrationContainerContext Context { get; set; } + } +} diff --git a/src/NuGet.Services.V3/Registration/Models/RegistrationLeaf.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationLeaf.cs new file mode 100644 index 000000000..1c5b49583 --- /dev/null +++ b/src/NuGet.Services.V3/Registration/Models/RegistrationLeaf.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Registration +{ + /// + /// Source: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-leaf + /// + public class RegistrationLeaf + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("@type")] + public List Types { get; set; } + + [JsonProperty("catalogEntry")] + public string CatalogEntry { get; set; } + + [JsonProperty("listed")] + public bool? Listed { get; set; } + + [JsonProperty("packageContent")] + public string PackageContent { get; set; } + + [JsonProperty("published")] + public DateTimeOffset? Published { get; set;} + + [JsonProperty("registration")] + public string Registration { get; set; } + + [JsonProperty("@context")] + public RegistrationLeafContext Context { get; set; } + } +} diff --git a/src/NuGet.Services.V3/Registration/Models/RegistrationLeafContext.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationLeafContext.cs new file mode 100644 index 000000000..a3ab98594 --- /dev/null +++ b/src/NuGet.Services.V3/Registration/Models/RegistrationLeafContext.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NuGet.Protocol.Catalog; + +namespace NuGet.Protocol.Registration +{ + public class RegistrationLeafContext + { + [JsonProperty("@vocab")] + public string Vocab { get; set; } + + [JsonProperty("xsd")] + public string Xsd { get; set; } + + [JsonProperty("catalogEntry")] + public ContextTypeDescription CatalogEntry { get; set; } + + [JsonProperty("registration")] + public ContextTypeDescription Registration { get; set; } + + [JsonProperty("packageContent")] + public ContextTypeDescription PackageContent { get; set; } + + [JsonProperty("published")] + public ContextTypeDescription Published { get; set; } + } +} diff --git a/src/NuGet.Services.V3/Registration/Models/RegistrationLeafItem.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationLeafItem.cs new file mode 100644 index 000000000..0debf6f0f --- /dev/null +++ b/src/NuGet.Services.V3/Registration/Models/RegistrationLeafItem.cs @@ -0,0 +1,35 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Registration +{ + /// + /// Source: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-leaf-object-in-a-page + /// + public class RegistrationLeafItem : ICommitted + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("@type")] + public string Type { get; set; } + + [JsonProperty("commitId")] + public string CommitId { get; set; } + + [JsonProperty("commitTimeStamp")] + public DateTimeOffset CommitTimestamp { get; set; } + + [JsonProperty("catalogEntry")] + public RegistrationCatalogEntry CatalogEntry { get; set; } + + [JsonProperty("packageContent")] + public string PackageContent { get; set; } + + [JsonProperty("registration")] + public string Registration { get; set; } + } +} diff --git a/src/NuGet.Services.V3/Registration/Models/RegistrationPackageDependency.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationPackageDependency.cs new file mode 100644 index 000000000..0c02d0b93 --- /dev/null +++ b/src/NuGet.Services.V3/Registration/Models/RegistrationPackageDependency.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NuGet.Protocol.Catalog; + +namespace NuGet.Protocol.Registration +{ + public class RegistrationPackageDependency : PackageDependency + { + [JsonProperty("registration", Order = 1)] + public string Registration { get; set; } + } +} diff --git a/src/NuGet.Services.V3/Registration/Models/RegistrationPackageDependencyGroup.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationPackageDependencyGroup.cs new file mode 100644 index 000000000..f0def4c36 --- /dev/null +++ b/src/NuGet.Services.V3/Registration/Models/RegistrationPackageDependencyGroup.cs @@ -0,0 +1,11 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Protocol.Catalog; + +namespace NuGet.Protocol.Registration +{ + public class RegistrationPackageDependencyGroup : BasePackageDependencyGroup + { + } +} diff --git a/src/NuGet.Services.V3/Registration/Models/RegistrationPackageVulnerability.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationPackageVulnerability.cs new file mode 100644 index 000000000..1c56d6f54 --- /dev/null +++ b/src/NuGet.Services.V3/Registration/Models/RegistrationPackageVulnerability.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace NuGet.Protocol.Registration +{ + public class RegistrationPackageVulnerability + { + [JsonProperty("advisoryUrl")] + public string AdvisoryUrl { get; set; } + + [JsonProperty("severity")] + public string Severity { get; set; } + } +} diff --git a/src/NuGet.Services.V3/Registration/Models/RegistrationPage.cs b/src/NuGet.Services.V3/Registration/Models/RegistrationPage.cs new file mode 100644 index 000000000..f9fdc0cf1 --- /dev/null +++ b/src/NuGet.Services.V3/Registration/Models/RegistrationPage.cs @@ -0,0 +1,53 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace NuGet.Protocol.Registration +{ + /// + /// This model is used for both the registration page item (found in a registration index) and for a registration + /// page fetched on its own. + /// Source: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-page + /// Source: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-page-object + /// + public class RegistrationPage : ICommitted + { + [JsonProperty("@id")] + public string Url { get; set; } + + [JsonProperty("@type")] + public string Type { get; set; } + + [JsonProperty("commitId")] + public string CommitId { get; set; } + + [JsonProperty("commitTimeStamp")] + public DateTimeOffset CommitTimestamp { get; set; } + + [JsonProperty("count")] + public int Count { get; set; } + + /// + /// This property can be null when this model is used as an item in when + /// the server decided not to inline the leaf items. In this case, the property can be used + /// fetch another instance with the property filled in. + /// + [JsonProperty("items")] + public List Items { get; set; } + + [JsonProperty("parent")] + public string Parent { get; set; } + + [JsonProperty("lower")] + public string Lower { get; set; } + + [JsonProperty("upper")] + public string Upper { get; set; } + + [JsonProperty("@context")] + public RegistrationContainerContext Context { get; set; } + } +} diff --git a/src/NuGet.Services.V3/Registration/RegistrationClient.cs b/src/NuGet.Services.V3/Registration/RegistrationClient.cs new file mode 100644 index 000000000..63357280d --- /dev/null +++ b/src/NuGet.Services.V3/Registration/RegistrationClient.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net; +using System.Threading.Tasks; +using NuGet.Protocol.Catalog; + +namespace NuGet.Protocol.Registration +{ + public class RegistrationClient : IRegistrationClient + { + private readonly ISimpleHttpClient _simpleHttpClient; + + public RegistrationClient(ISimpleHttpClient simpleHttpClient) + { + _simpleHttpClient = simpleHttpClient ?? throw new ArgumentNullException(nameof(simpleHttpClient)); + } + + public async Task GetIndexOrNullAsync(string indexUrl) + { + var result = await _simpleHttpClient.DeserializeUrlAsync(indexUrl); + if (result.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + + return result.GetResultOrThrow(); + } + + public async Task GetPageAsync(string pageUrl) + { + var result = await _simpleHttpClient.DeserializeUrlAsync(pageUrl); + return result.GetResultOrThrow(); + } + + public async Task GetLeafAsync(string leafUrl) + { + var result = await _simpleHttpClient.DeserializeUrlAsync(leafUrl); + return result.GetResultOrThrow(); + } + } +} diff --git a/src/NuGet.Services.V3/Registration/RegistrationUrlBuilder.cs b/src/NuGet.Services.V3/Registration/RegistrationUrlBuilder.cs new file mode 100644 index 000000000..8daaf0b07 --- /dev/null +++ b/src/NuGet.Services.V3/Registration/RegistrationUrlBuilder.cs @@ -0,0 +1,45 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Versioning; + +namespace NuGet.Protocol.Registration +{ + public static class RegistrationUrlBuilder + { + /// + /// Builds a URL to a registration index. This pattern is documented. + /// Source: https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource#registration-pages-and-leaves + /// + /// The base URL of the registration hive. Trailing slashes will be stripped. + /// The package ID. + public static string GetIndexUrl(string baseUrl, string id) + { + var normalizedBaseUrl = NormalizeBaseUrl(baseUrl); + var lowerId = id.ToLowerInvariant(); + return $"{normalizedBaseUrl}{lowerId}/index.json"; + } + + /// + /// Builds a URL to a registration leaf. This is a document that is specific to a package ID and version. Note + /// that this URL in general should be discovered using the registration index. In the case of NuGet.org, the + /// leaf URL has a specific pattern than can be generated without a lookup. + /// + /// The base URL of the registration hive. Trailing slashes will be stripped. + /// The package ID. + /// The package version. It will be parsed and normalized. + public static string GetLeafUrl(string baseUrl, string id, string version) + { + var normalizedBaseUrl = NormalizeBaseUrl(baseUrl); + var lowerId = id.ToLowerInvariant(); + var parsedVersion = NuGetVersion.Parse(version); + var lowerVersion = parsedVersion.ToNormalizedString().ToLowerInvariant(); + return $"{normalizedBaseUrl}{lowerId}/{lowerVersion}.json"; + } + + private static string NormalizeBaseUrl(string baseUrl) + { + return baseUrl.TrimEnd('/') + '/'; + } + } +} diff --git a/src/NuGet.Services.V3/Support/Guard.cs b/src/NuGet.Services.V3/Support/Guard.cs new file mode 100644 index 000000000..83cc15d4c --- /dev/null +++ b/src/NuGet.Services.V3/Support/Guard.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services +{ + public static class Guard + { + /// + /// We could use here, but it's preferable in this + /// case to even fail on a non-Debug build. The goal of this method is to allow the implementor to embed more + /// intent into the implementation so that future code changes are less likely to introduce bugs and so that + /// the implementation makes more sense to future readers. This is, of course, in addition to unit test + /// coverage. + /// + public static void Assert(bool condition, string message) + { + if (!condition) + { + throw new InvalidOperationException(message); + } + } + + /// + /// Similar to but run in Release builds. + /// + public static void Fail(string message) + { + Assert(false, message); + } + } +} diff --git a/src/NuGet.Services.V3/Support/IdAndValue.cs b/src/NuGet.Services.V3/Support/IdAndValue.cs new file mode 100644 index 000000000..831ab6125 --- /dev/null +++ b/src/NuGet.Services.V3/Support/IdAndValue.cs @@ -0,0 +1,19 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NuGet.Services +{ + public class IdAndValue + { + public IdAndValue(string id, T value) + { + Id = id ?? throw new ArgumentNullException(nameof(id)); + Value = value; + } + + public string Id { get; } + public T Value { get; } + } +} diff --git a/src/NuGet.Services.V3/Support/KeyValuePair.cs b/src/NuGet.Services.V3/Support/KeyValuePair.cs new file mode 100644 index 000000000..177f08df0 --- /dev/null +++ b/src/NuGet.Services.V3/Support/KeyValuePair.cs @@ -0,0 +1,15 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; + +namespace NuGet.Services +{ + public static class KeyValuePair + { + public static KeyValuePair Create(TKey key, TValue value) + { + return new KeyValuePair(key, value); + } + } +} diff --git a/src/NuGet.Services.V3/Support/ReferenceEqualityComparer.cs b/src/NuGet.Services.V3/Support/ReferenceEqualityComparer.cs new file mode 100644 index 000000000..366baf7c5 --- /dev/null +++ b/src/NuGet.Services.V3/Support/ReferenceEqualityComparer.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace NuGet.Services +{ + /// + /// Source: https://stackoverflow.com/a/35520207 + /// + public sealed class ReferenceEqualityComparer : IEqualityComparer where T : class + { + public static readonly ReferenceEqualityComparer Default = new ReferenceEqualityComparer(); + + private ReferenceEqualityComparer() + { + } + + [DebuggerStepThrough] + public bool Equals(T x, T y) + { + return ReferenceEquals(x, y); + } + + [DebuggerStepThrough] + public int GetHashCode(T obj) + { + return RuntimeHelpers.GetHashCode(obj); + } + } +} diff --git a/src/NuGet.Services.V3/V3TelemetryService.cs b/src/NuGet.Services.V3/V3TelemetryService.cs new file mode 100644 index 000000000..89a35cac5 --- /dev/null +++ b/src/NuGet.Services.V3/V3TelemetryService.cs @@ -0,0 +1,39 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Services.FeatureFlags; +using NuGet.Services.Logging; + +namespace NuGet.Services.V3 +{ + public class V3TelemetryService : IV3TelemetryService, IFeatureFlagTelemetryService + { + private const string Prefix = "V3."; + + private readonly ITelemetryClient _telemetryClient; + + public V3TelemetryService(ITelemetryClient telemetryClient) + { + _telemetryClient = telemetryClient ?? throw new ArgumentNullException(nameof(telemetryClient)); + } + + public IDisposable TrackCatalogLeafDownloadBatch(int count) + { + return _telemetryClient.TrackDuration( + Prefix + "CatalogLeafDownloadBatchSeconds", + new Dictionary + { + { "Count", count.ToString() }, + }); + } + + public void TrackFeatureFlagStaleness(TimeSpan staleness) + { + _telemetryClient.TrackMetric( + Prefix + "FeatureFlagStalenessSeconds", + staleness.TotalSeconds); + } + } +} diff --git a/src/PackageHash/PackageHash.csproj b/src/PackageHash/PackageHash.csproj index 84d5aef93..8cccc683e 100644 --- a/src/PackageHash/PackageHash.csproj +++ b/src/PackageHash/PackageHash.csproj @@ -76,7 +76,7 @@ - 2.74.0 + 2.75.0 diff --git a/src/PackageLagMonitor/Monitoring.PackageLag.csproj b/src/PackageLagMonitor/Monitoring.PackageLag.csproj index ea217fbec..99ae4f020 100644 --- a/src/PackageLagMonitor/Monitoring.PackageLag.csproj +++ b/src/PackageLagMonitor/Monitoring.PackageLag.csproj @@ -85,6 +85,10 @@ {4b4b1efb-8f33-42e6-b79f-54e7f3293d31} NuGet.Jobs.Common + + {d44c2e89-2d98-44bd-8712-8ccbe4e67c9c} + NuGet.Protocol.Catalog + @@ -102,9 +106,6 @@ 2.2.0 - - 4.1.0-master-3566788 - 0.3.0 runtime; build; native; contentfiles; analyzers diff --git a/src/README.md b/src/README.md new file mode 100644 index 000000000..83f808cbf --- /dev/null +++ b/src/README.md @@ -0,0 +1,15 @@ +# NuGet V3 Source Code + +This repo contains nuget.org's implementation of the [NuGet V3 API](https://docs.microsoft.com/en-us/nuget/api/overview). + +The following folders power the [search](https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource) and [autocomplete](https://docs.microsoft.com/en-us/nuget/api/search-autocomplete-service-resource) resources: + +* `NuGet.Jobs.Auxiliary2AzureSearch` - The job that updates miscellaneous data in the Azure Search index. +* `NuGet.Jobs.Catalog2AzureSearch` - The job that updates the Azure Search index when packages are uploaded or modified. +* `NuGet.Jobs.Db2AzureSearch` - The job that creates the Azure Search index using the Gallery database. +* `NuGet.Services.SearchService` - The nuget.org search service, powered by Azure Search. + +Other interesting folders include: + +* `Ng` - The job that updates several V3 resources, including the [catalog](https://docs.microsoft.com/en-us/nuget/api/catalog-resource) and [package content](https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource) resources. +* `NuGet.Jobs.Catalog2Registration` - The job that updates the [package metadata](https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource) resource. \ No newline at end of file diff --git a/src/SplitLargeFiles/SplitLargeFiles.csproj b/src/SplitLargeFiles/SplitLargeFiles.csproj index 88e40ee9e..bd1c3e7e4 100644 --- a/src/SplitLargeFiles/SplitLargeFiles.csproj +++ b/src/SplitLargeFiles/SplitLargeFiles.csproj @@ -59,7 +59,7 @@ - 2.74.0 + 2.75.0 9.3.3 diff --git a/src/StatusAggregator/StatusAggregator.csproj b/src/StatusAggregator/StatusAggregator.csproj index 811ece0ac..f5b1f65a6 100644 --- a/src/StatusAggregator/StatusAggregator.csproj +++ b/src/StatusAggregator/StatusAggregator.csproj @@ -160,13 +160,13 @@ 2.2.0 - 2.74.0 + 2.75.0 - 2.74.0 + 2.75.0 - 2.74.0 + 2.75.0 0.3.0 diff --git a/src/Validation.Common.Job/Validation.Common.Job.csproj b/src/Validation.Common.Job/Validation.Common.Job.csproj index 3609513e0..90d2fcf71 100644 --- a/src/Validation.Common.Job/Validation.Common.Job.csproj +++ b/src/Validation.Common.Job/Validation.Common.Job.csproj @@ -116,10 +116,10 @@ 5.0.0-preview1.5707 - 2.74.0 + 2.75.0 - 2.74.0 + 2.75.0 0.3.0 diff --git a/src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj b/src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj index e55dcbc3b..dd4f73ee4 100644 --- a/src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj +++ b/src/Validation.ScanAndSign.Core/Validation.ScanAndSign.Core.csproj @@ -65,7 +65,7 @@ all - 2.74.0 + 2.75.0 0.3.0 diff --git a/test.ps1 b/test.ps1 index 9af326e88..4aa7c06ad 100644 --- a/test.ps1 +++ b/test.ps1 @@ -22,7 +22,7 @@ Function Run-Tests { Trace-Log 'Running tests' - $xUnitExe = (Join-Path $PSScriptRoot "packages\xunit.runner.console\tools\xunit.console.exe") + $xUnitExe = (Join-Path $PSScriptRoot "packages\xunit.runner.console.2.1.0\tools\xunit.console.exe") $TestAssemblies = ` "tests\Monitoring.PackageLag.Tests\bin\$Configuration\Monitoring.PackageLag.Tests.dll", ` @@ -30,7 +30,7 @@ Function Run-Tests { "tests\NuGet.Jobs.GitHubIndexer.Tests\bin\$Configuration\NuGet.Jobs.GitHubIndexer.Tests.dll", ` "tests\NuGet.Services.Revalidate.Tests\bin\$Configuration\NuGet.Services.Revalidate.Tests.dll", ` "tests\NuGet.Services.Validation.Orchestrator.Tests\bin\$Configuration\NuGet.Services.Validation.Orchestrator.Tests.dll", ` - "tests\StatusAggregator\bin\$Configuration\StatusAggregator.dll", ` + "tests\StatusAggregator.Tests\bin\$Configuration\StatusAggregator.Tests.dll", ` "tests\Tests.CredentialExpiration\bin\$Configuration\Tests.CredentialExpiration.dll", ` "tests\Tests.Gallery.Maintenance\bin\$Configuration\Tests.Gallery.Maintenance.dll", ` "tests\Tests.Stats.AggregateCdnDownloadsInGallery\bin\$Configuration\Tests.Stats.AggregateCdnDownloadsInGallery.dll", ` @@ -45,14 +45,27 @@ Function Run-Tests { "tests\Validation.PackageSigning.RevalidateCertificate.Tests\bin\$Configuration\Validation.PackageSigning.RevalidateCertificate.Tests.dll", ` "tests\Validation.PackageSigning.ScanAndSign.Tests\bin\$Configuration\Validation.PackageSigning.ScanAndSign.Tests.dll", ` "tests\Validation.PackageSigning.ValidateCertificate.Tests\bin\$Configuration\Validation.PackageSigning.ValidateCertificate.Tests.dll", ` - "tests\Validation.Symbols.Tests\bin\$Configuration\Validation.Symbols.Core.Tests.dll", ` + "tests\Validation.Symbols.Core.Tests\bin\$Configuration\Validation.Symbols.Core.Tests.dll", ` "tests\Validation.Symbols.Tests\bin\$Configuration\Validation.Symbols.Tests.dll", ` - "tests\SplitLargeFiles.Tests\bin\$Configuration\NuGet.Tools.SplitLargeFiles.Tests.dll" - + "tests\SplitLargeFiles.Tests\bin\$Configuration\NuGet.Tools.SplitLargeFiles.Tests.dll", ` + "tests\NgTests\bin\$Configuration\NgTests.dll", ` + "tests\CatalogTests\bin\$Configuration\CatalogTests.dll", ` + "tests\CatalogMetadataTests\bin\$Configuration\CatalogMetadataTests.dll", ` + "tests\NuGet.Protocol.Catalog.Tests\bin\$Configuration\NuGet.Protocol.Catalog.Tests.dll", ` + "tests\NuGet.Services.AzureSearch.Tests\bin\$Configuration\NuGet.Services.AzureSearch.Tests.dll", ` + "tests\NuGet.Services.SearchService.Tests\bin\$Configuration\NuGet.Services.SearchService.Tests.dll", ` + "tests\NuGet.Jobs.Catalog2Registration.Tests\bin\$Configuration\NuGet.Jobs.Catalog2Registration.Tests.dll" + $TestCount = 0 foreach ($Test in $TestAssemblies) { - & $xUnitExe (Join-Path $PSScriptRoot $Test) -xml "Results.$TestCount.xml" + $TestResultFile = "Results.$TestCount.xml" + & $xUnitExe (Join-Path $PSScriptRoot $Test) -xml $TestResultFile + if (-not (Test-Path $TestResultFile)) + { + Write-Error "The test run failed to produce a result file"; + exit 1; + } $TestCount++ } } @@ -85,4 +98,4 @@ if ($BuildErrors) { Error-Log "Tests completed with $($BuildErrors.Count) error(s):`r`n$($ErrorLines -join "`r`n")" -Fatal } -Write-Host ("`r`n" * 3) +Write-Host ("`r`n" * 3) \ No newline at end of file diff --git a/tests/BasicSearchTests.FunctionalTests.Core/BaseFunctionalTests.cs b/tests/BasicSearchTests.FunctionalTests.Core/BaseFunctionalTests.cs new file mode 100644 index 000000000..271e89ecc --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/BaseFunctionalTests.cs @@ -0,0 +1,35 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using BasicSearchTests.FunctionalTests.Core.TestSupport; +using System.Net; + +namespace BasicSearchTests.FunctionalTests.Core +{ + public class BaseFunctionalTests : IDisposable + { + protected HttpClient Client; + protected RetryHandler RetryHandler; + + public BaseFunctionalTests() + : this(EnvironmentSettings.SearchServiceBaseUrl) + { + } + + public BaseFunctionalTests(string baseUrl) + { + // Arrange + ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; + RetryHandler = new RetryHandler(new HttpClientHandler()); + Client = new HttpClient(RetryHandler) { BaseAddress = new Uri(baseUrl) }; + } + + public void Dispose() + { + Client.Dispose(); + RetryHandler.Dispose(); + } + } +} \ No newline at end of file diff --git a/tests/BasicSearchTests.FunctionalTests.Core/BasicSearchTests.FunctionalTests.Core.csproj b/tests/BasicSearchTests.FunctionalTests.Core/BasicSearchTests.FunctionalTests.Core.csproj new file mode 100644 index 000000000..c0dcf261e --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/BasicSearchTests.FunctionalTests.Core.csproj @@ -0,0 +1,99 @@ + + + + + Debug + AnyCPU + {EEA7B6C1-0358-4E67-9D2A-E30B8FF9FF3D} + Library + Properties + BasicSearchTests.FunctionalTests.Core + BasicSearchTests.FunctionalTests.Core + v4.7.2 + 512 + + + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 5.2.3 + + + 9.0.1 + + + 2.4.1 + + + 2.4.1 + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + \ No newline at end of file diff --git a/tests/BasicSearchTests.FunctionalTests.Core/Constants.cs b/tests/BasicSearchTests.FunctionalTests.Core/Constants.cs new file mode 100644 index 000000000..a15ff2962 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/Constants.cs @@ -0,0 +1,30 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace BasicSearchTests.FunctionalTests.Core +{ + public static class Constants + { + // Predefined Texts + public const string TestPackageId = "BaseTestPackage"; + public const string TestPackageId_SearchFilters = "BaseTestPackage.SearchFilters"; + public const string TestPackageId_SemVer2 = "BaseTestPackage.SemVer2"; + public const string TestPackageId_Unlisted = "BaseTestPackage.Unlisted"; + public const string TestPackageId_PackageType = "BaseTestPackage.DotnetTool"; + public const string TestPackageVersion = "1.0.0"; + public const string TestPackageVersion_SearchFilters_Default = "1.1.0"; + public const string TestPackageVersion_SearchFilters_Prerel = "1.2.0-beta"; + public const string TestPackageVersion_SearchFilters_SemVer2 = "1.3.0+metadata"; + public const string TestPackageVersion_SearchFilters_PrerelSemVer2 = "1.4.0-delta.4"; + public const string TestPackageVersion_Unlisted = "1.1.0"; + public const string TestPackageVersion_PackageType = "1.0.0"; + public const string TestPackageTitle = "BaseTestPackage"; + public const string TestPackageDescription = "Package description"; + public const string TestPackageSummary = ""; + public const string TestPackageAuthor = "clayco"; + public const string TestPackageCopyright = "Copyright 2013"; + public const string NonExistentSearchString = "asadfasdfsadfwerfasdf23423rafdsf"; + public const string TestPackageOwner = "NugetTestAccount"; + public const string TestPackageTag = "json"; + } +} \ No newline at end of file diff --git a/tests/BasicSearchTests.FunctionalTests.Core/EnvironmentSettings.cs b/tests/BasicSearchTests.FunctionalTests.Core/EnvironmentSettings.cs new file mode 100644 index 000000000..5653bb008 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/EnvironmentSettings.cs @@ -0,0 +1,62 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace BasicSearchTests.FunctionalTests.Core +{ + /// + /// This class reads the various test run settings which are set through env variable. + /// + public class EnvironmentSettings + { + private static string _searchServiceBaseurl; + + /// + /// The environment against which the (search service) test has to be run. The value would be picked from env variable. + /// If nothing is specified, int search service is used as default. + /// + public static string SearchServiceBaseUrl + { + get + { + if (string.IsNullOrEmpty(_searchServiceBaseurl)) + { + _searchServiceBaseurl = GetEnvironmentVariable("SearchServiceUrl", "https://nuget-int-0-v2v3search.cloudapp.net/"); + } + + return _searchServiceBaseurl; + } + } + + public const string ConfigurationFilePathVariableName = "ConfigurationFilePath"; + + public static string ConfigurationFilePath => GetEnvironmentVariable(ConfigurationFilePathVariableName, required: true); + + private static string GetEnvironmentVariable(string key, string defaultValue = null, bool required = false) + { + var envVariable = Environment.GetEnvironmentVariable(key, EnvironmentVariableTarget.User); + if (string.IsNullOrEmpty(envVariable)) + { + envVariable = Environment.GetEnvironmentVariable(key, EnvironmentVariableTarget.Process); + } + + if (string.IsNullOrEmpty(envVariable)) + { + envVariable = Environment.GetEnvironmentVariable(key, EnvironmentVariableTarget.Machine); + } + + if (string.IsNullOrEmpty(envVariable)) + { + if (required) + { + throw new InvalidOperationException($"The '{key}' environment variable must be set to run this test"); + } + + envVariable = defaultValue; + } + + return envVariable; + } + } +} diff --git a/tests/BasicSearchTests.FunctionalTests.Core/Models/AtContext.cs b/tests/BasicSearchTests.FunctionalTests.Core/Models/AtContext.cs new file mode 100644 index 000000000..542c335b5 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/Models/AtContext.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace BasicSearchTests.FunctionalTests.Core.Models +{ + public class AtContext + { + [JsonProperty("@vocab")] + public string AtVocab { get; set; } + + [JsonProperty("@base")] + public string AtBase { get; set; } + } +} \ No newline at end of file diff --git a/tests/BasicSearchTests.FunctionalTests.Core/Models/AutocompleteResult.cs b/tests/BasicSearchTests.FunctionalTests.Core/Models/AutocompleteResult.cs new file mode 100644 index 000000000..68abe53f2 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/Models/AutocompleteResult.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace BasicSearchTests.FunctionalTests.Core.Models +{ + public class AutocompleteResult + { + public int? TotalHits { get; set; } + + public List Data { get; set; } + } +} \ No newline at end of file diff --git a/tests/BasicSearchTests.FunctionalTests.Core/Models/PackageVersion.cs b/tests/BasicSearchTests.FunctionalTests.Core/Models/PackageVersion.cs new file mode 100644 index 000000000..43f002847 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/Models/PackageVersion.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; + +namespace BasicSearchTests.FunctionalTests.Core.Models +{ + public class PackageVersion + { + public string Version; + + public long Downloads; + + [JsonProperty("@id")] + public string AtId; + } +} \ No newline at end of file diff --git a/tests/BasicSearchTests.FunctionalTests.Core/Models/SearchResult.cs b/tests/BasicSearchTests.FunctionalTests.Core/Models/SearchResult.cs new file mode 100644 index 000000000..880ec54e6 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/Models/SearchResult.cs @@ -0,0 +1,10 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace BasicSearchTests.FunctionalTests.Core.Models +{ + public class SearchResult + { + public int? TotalHits { get; set; } + } +} diff --git a/tests/BasicSearchTests.FunctionalTests.Core/Models/V2PackageRegistration.cs b/tests/BasicSearchTests.FunctionalTests.Core/Models/V2PackageRegistration.cs new file mode 100644 index 000000000..2f05faf53 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/Models/V2PackageRegistration.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +namespace BasicSearchTests.FunctionalTests.Core.Models +{ + public class V2PackageRegistration + { + public string Id { get; set; } + + public long DownloadCount { get; set; } + + public IList Owners { get; set; } + } +} diff --git a/tests/BasicSearchTests.FunctionalTests.Core/Models/V2SearchResult.cs b/tests/BasicSearchTests.FunctionalTests.Core/Models/V2SearchResult.cs new file mode 100644 index 000000000..670eb37a6 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/Models/V2SearchResult.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; + +namespace BasicSearchTests.FunctionalTests.Core.Models +{ + public class V2SearchResult: SearchResult + { + public IList Data { get; set; } + } +} \ No newline at end of file diff --git a/tests/BasicSearchTests.FunctionalTests.Core/Models/V2SearchResultEntry.cs b/tests/BasicSearchTests.FunctionalTests.Core/Models/V2SearchResultEntry.cs new file mode 100644 index 000000000..aec4c9a44 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/Models/V2SearchResultEntry.cs @@ -0,0 +1,71 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; + +namespace BasicSearchTests.FunctionalTests.Core.Models +{ + public class V2SearchResultEntry + { + public V2PackageRegistration PackageRegistration { get; set; } + + [JsonProperty(Required = Required.Always)] + public string Version { get; set; } + + [JsonProperty(Required = Required.Always)] + public string NormalizedVersion { get; set; } + + public string Title { get; set; } + + public string Description { get; set; } + + public string Summary { get; set; } + + public string Authors { get; set; } + + public string Copyright { get; set; } + + public string Language { get; set; } + + public string Tags { get; set; } + + public string ReleaseNotes { get; set; } + + public string ProjectUrl { get; set; } + + public string IconUrl { get; set; } + + public bool IsLatestStable { get; set; } + + public bool IsLatest { get; set; } + + public bool Listed { get; set; } + + public DateTime Created { get; set; } + + public DateTime Published { get; set; } + + public DateTime LastUpdated { get; set; } + + public DateTime? LastEdited { get; set; } + + public long DownloadCount { get; set; } + + public string FlattenedDependencies { get; set; } + + public object[] Dependencies { get; set; } + + public string[] SupportedFrameworks { get; set; } + + public string Hash { get; set; } + + public string HashAlgorithm { get; set; } + + public long PackageFileSize { get; set; } + + public string LicenseUrl { get; set; } + + public bool RequireslicenseAcceptance { get; set; } + } +} diff --git a/tests/BasicSearchTests.FunctionalTests.Core/Models/V3SearchResult.cs b/tests/BasicSearchTests.FunctionalTests.Core/Models/V3SearchResult.cs new file mode 100644 index 000000000..34c77a1d8 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/Models/V3SearchResult.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace BasicSearchTests.FunctionalTests.Core.Models +{ + public class V3SearchResult: SearchResult + { + [JsonProperty("@context")] + public AtContext AtContext { get; set; } + + public IList Data { get; set; } + } +} \ No newline at end of file diff --git a/tests/BasicSearchTests.FunctionalTests.Core/Models/V3SearchResultEntry.cs b/tests/BasicSearchTests.FunctionalTests.Core/Models/V3SearchResultEntry.cs new file mode 100644 index 000000000..e18dacb27 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/Models/V3SearchResultEntry.cs @@ -0,0 +1,47 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace BasicSearchTests.FunctionalTests.Core.Models +{ + public class V3SearchResultEntry + { + [JsonProperty("@id")] + public string AtId { get; set; } + + [JsonProperty("@type")] + public string AtType { get; set; } + + public string Registration { get; set; } + + [JsonProperty(Required = Required.Always)] + public string Id { get; set; } + + [JsonProperty(Required = Required.Always)] + public string Version { get; set; } + + public string Description { get; set; } + + public string Summary { get; set; } + + public string Title { get; set; } + + public string IconUrl { get; set; } + + public string LicenseUrl { get; set; } + + public string ProjectUrl { get; set; } + + public string[] Tags { get; set; } + + public string[] Authors { get; set; } + + public long TotalDownloads { get; set; } + + public List PackageTypes { get; set; } + + public PackageVersion[] Versions { get; set; } + } +} diff --git a/tests/BasicSearchTests.FunctionalTests.Core/Models/V3SearchResultPackageType.cs b/tests/BasicSearchTests.FunctionalTests.Core/Models/V3SearchResultPackageType.cs new file mode 100644 index 000000000..7e7f749d7 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/Models/V3SearchResultPackageType.cs @@ -0,0 +1,10 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace BasicSearchTests.FunctionalTests.Core.Models +{ + public class V3SearchResultPackageType + { + public string Name { get; set; } + } +} diff --git a/tests/BasicSearchTests.FunctionalTests.Core/Properties/AssemblyInfo.cs b/tests/BasicSearchTests.FunctionalTests.Core/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..af70f9b3a --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("BasicSearchTests.FunctionalTests.Core")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("BasicSearchTests.FunctionalTests.Core")] +[assembly: AssemblyCopyright("Copyright © 2015")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("af189f05-efc3-4a98-91c7-2af1e4b8b131")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/AutocompleteBuilder.cs b/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/AutocompleteBuilder.cs new file mode 100644 index 000000000..96a4f7a5a --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/AutocompleteBuilder.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Specialized; +using System.Web; + +namespace BasicSearchTests.FunctionalTests.Core.TestSupport +{ + public class AutocompleteBuilder : QueryBuilder + { + public int? Skip { get; set; } + + public int? Take { get; set; } + + public bool IncludeSemVer2 { get; set; } + + public string PackageType { get; set; } + + public AutocompleteBuilder() : base("/autocomplete?") { } + + protected override NameValueCollection GetQueryString() + { + var queryString = HttpUtility.ParseQueryString(string.Empty); + queryString["q"] = Query; + queryString["prerelease"] = Prerelease.ToString(); + + if (Skip.HasValue) + { + queryString["skip"] = Skip.ToString(); + } + + if (Take.HasValue) + { + queryString["take"] = Take.ToString(); + } + + if (IncludeSemVer2) + { + queryString["semVerLevel"] = "2.0.0"; + } + + if (PackageType != null) + { + queryString["packageType"] = PackageType; + } + + return queryString; + } + } +} \ No newline at end of file diff --git a/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/QueryBuilder.cs b/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/QueryBuilder.cs new file mode 100644 index 000000000..5dd9be231 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/QueryBuilder.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Specialized; + +namespace BasicSearchTests.FunctionalTests.Core.TestSupport +{ + public class QueryBuilder + { + protected string Endpoint; + + public string Query { get; set; } + public bool Prerelease { get; set; } + + public QueryBuilder(string endpoint) + { + Endpoint = endpoint; + } + + protected virtual NameValueCollection GetQueryString() + { + var queryString = System.Web.HttpUtility.ParseQueryString(string.Empty); + queryString["q"] = Query; + queryString["prerelease"] = Prerelease.ToString(); + return queryString; + } + + public Uri RequestUri + { + get + { + return new Uri(Endpoint + GetQueryString(), UriKind.Relative); + } + } + } +} \ No newline at end of file diff --git a/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/RetryHandler.cs b/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/RetryHandler.cs new file mode 100644 index 000000000..1c2cef24f --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/RetryHandler.cs @@ -0,0 +1,38 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace BasicSearchTests.FunctionalTests.Core.TestSupport +{ + public class RetryHandler : DelegatingHandler + { + private const int MaxRetries = 3; + + public RetryHandler(HttpMessageHandler innerHandler) + : base(innerHandler) + { } + + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + HttpResponseMessage response = null; + for (int i = 0; i < MaxRetries; i++) + { + response?.Dispose(); + + response = await base.SendAsync(request, cancellationToken); + // One of the test validates that we get a 404 for a route, don't block on NotFound Http status code either. + if (response.IsSuccessStatusCode || response.StatusCode == System.Net.HttpStatusCode.NotFound) + { + return response; + } + } + + return response; + } + } +} \ No newline at end of file diff --git a/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/V2SearchBuilder.cs b/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/V2SearchBuilder.cs new file mode 100644 index 000000000..90f8d8926 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/V2SearchBuilder.cs @@ -0,0 +1,70 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Specialized; +using System.Web; + +namespace BasicSearchTests.FunctionalTests.Core.TestSupport +{ + public class V2SearchBuilder : QueryBuilder + { + public bool IgnoreFilter { get; set; } + + public int? Skip { get; set; } + + public int? Take { get; set; } + + public bool CountOnly { get; set; } + + public bool IncludeSemVer2 { get; set; } + + public string SortBy { get; set; } + + public bool? LuceneQuery { get; set; } + + public string PackageType { get; set; } + + public V2SearchBuilder() : base("/search/query?") { } + + protected override NameValueCollection GetQueryString() + { + var queryString = HttpUtility.ParseQueryString(string.Empty); + queryString["q"] = Query; + queryString["prerelease"] = Prerelease.ToString(); + queryString["ignoreFilter"] = IgnoreFilter.ToString(); + queryString["CountOnly"] = CountOnly.ToString(); + + if (Skip.HasValue) + { + queryString["Skip"] = Skip.ToString(); + } + + if (Take.HasValue) + { + queryString["Take"] = Take.ToString(); + } + + if (IncludeSemVer2) + { + queryString["semVerLevel"] = "2.0.0"; + } + + if (!string.IsNullOrWhiteSpace(SortBy)) + { + queryString["sortBy"] = SortBy; + } + + if (LuceneQuery.HasValue) + { + queryString["luceneQuery"] = LuceneQuery.ToString(); + } + + if (PackageType != null) + { + queryString["packageType"] = PackageType; + } + + return queryString; + } + } +} \ No newline at end of file diff --git a/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/V3SearchBuilder.cs b/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/V3SearchBuilder.cs new file mode 100644 index 000000000..67f26bd66 --- /dev/null +++ b/tests/BasicSearchTests.FunctionalTests.Core/TestSupport/V3SearchBuilder.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Specialized; +using System.Web; + +namespace BasicSearchTests.FunctionalTests.Core.TestSupport +{ + public class V3SearchBuilder : QueryBuilder + { + public int? Skip { get; set; } + + public int? Take { get; set; } + + public bool IncludeSemVer2 { get; set; } + + public string PackageType { get; set; } + + public V3SearchBuilder() : base("/query?") { } + + protected override NameValueCollection GetQueryString() + { + var queryString = HttpUtility.ParseQueryString(string.Empty); + queryString["q"] = Query; + queryString["prerelease"] = Prerelease.ToString(); + + if (Skip.HasValue) + { + queryString["skip"] = Skip.ToString(); + } + + if (Take.HasValue) + { + queryString["take"] = Take.ToString(); + } + + if (IncludeSemVer2) + { + queryString["semVerLevel"] = "2.0.0"; + } + + if (PackageType != null) + { + queryString["packageType"] = PackageType; + } + + return queryString; + } + } +} \ No newline at end of file diff --git a/tests/CatalogMetadataTests/AzureStorageFacts.cs b/tests/CatalogMetadataTests/AzureStorageFacts.cs new file mode 100644 index 000000000..c12b2c555 --- /dev/null +++ b/tests/CatalogMetadataTests/AzureStorageFacts.cs @@ -0,0 +1,150 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Collections.Generic; +using System.Threading.Tasks; +using Moq; +using Xunit; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace CatalogMetadataTests +{ + public class AzureStorageFacts : AzureStorageBaseFacts + { + public AzureStorageFacts() : base() + { + } + + [Theory] + [InlineData(true, true, "SHA512Value1", true, true, "SHA512Value1", true)] + [InlineData(true, true, "SHA512Value1", true, true, "SHA512Value2", false)] + [InlineData(false, false, null, true, true, "SHA512Value1", true)] + [InlineData(true, true, "SHA512Value1", false, false, null, false)] + [InlineData(false, false, null, false, false, null, true)] + [InlineData(true, false, null, true, true, "SHA512Value1", false)] + [InlineData(true, true, "SHA512Value1", true, false, null, false)] + [InlineData(true, false, null, true, false, null, false)] + public async void ValidateAreSynchronizedmethod(bool sourceBlobExists, + bool hasSourceBlobSHA512Value, + string sourceBlobSHA512Value, + bool destinationBlobExists, + bool hasDestinationBlobSHA512Value, + string destinationBlobSHA512Value, + bool expected) + { + // Arrange + var sourceBlob = GetMockedBlockBlob(sourceBlobExists, hasSourceBlobSHA512Value, sourceBlobSHA512Value, new Uri("https://blockBlob1")); + var destinationBlob = GetMockedBlockBlob(destinationBlobExists, hasDestinationBlobSHA512Value, destinationBlobSHA512Value, new Uri("https://blockBlob2")); + + // Act + var isSynchronized = await _storage.AreSynchronized(sourceBlob.Object, destinationBlob.Object); + + // Assert + sourceBlob.Verify(x => x.ExistsAsync(CancellationToken.None), Times.Once); + destinationBlob.Verify(x => x.ExistsAsync(CancellationToken.None), Times.Once); + + if (sourceBlobExists && destinationBlobExists) + { + sourceBlob.Verify(x => x.GetMetadataAsync(CancellationToken.None), Times.Once); + destinationBlob.Verify(x => x.GetMetadataAsync(CancellationToken.None), Times.Once); + + if (!hasSourceBlobSHA512Value) + { + sourceBlob.Verify(x => x.Uri, Times.Once); + } + if (!hasDestinationBlobSHA512Value) + { + destinationBlob.Verify(x => x.Uri, Times.Once); + } + if (hasSourceBlobSHA512Value && hasDestinationBlobSHA512Value) + { + sourceBlob.Verify(x => x.Uri, Times.Once); + destinationBlob.Verify(x => x.Uri, Times.Once); + } + } + + Assert.Equal(expected, isSynchronized); + } + + private Mock GetMockedBlockBlob(bool isExisted, bool hasSHA512Value, string SHA512Value, Uri blockBlobUri) + { + var mockBlob = new Mock(); + + mockBlob.Setup(x => x.ExistsAsync(CancellationToken.None)) + .ReturnsAsync(isExisted); + + if (isExisted) + { + mockBlob.Setup(x => x.Uri).Returns(blockBlobUri); + + if (hasSHA512Value) + { + mockBlob.Setup(x => x.GetMetadataAsync(CancellationToken.None)) + .ReturnsAsync(new Dictionary() + { + { "SHA512", SHA512Value } + }); + } + else + { + mockBlob.Setup(x => x.GetMetadataAsync(CancellationToken.None)) + .ReturnsAsync(new Dictionary()); + } + } + + return mockBlob; + } + + [Theory] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public async void ValidateAreSynchronizedmethodWithNullMetadata(bool isSourceBlobMetadataExisted, bool isDestinationBlobMetadataExists) + { + // Arrange + var sourceBlob = GetMockedBlockBlobWithNullMetadata(isSourceBlobMetadataExisted); + var destinationBlob = GetMockedBlockBlobWithNullMetadata(isDestinationBlobMetadataExists); + + // Act and Assert + Assert.False(await _storage.AreSynchronized(sourceBlob, destinationBlob)); + } + + private ICloudBlockBlob GetMockedBlockBlobWithNullMetadata(bool isBlobMetadataExisted) + { + if (isBlobMetadataExisted) + { + return Mock.Of(x => x.ExistsAsync(CancellationToken.None) == Task.FromResult(true) && + x.GetMetadataAsync(CancellationToken.None) == Task.FromResult>(new Dictionary())); + } + else + { + return Mock.Of(x => x.ExistsAsync(CancellationToken.None) == Task.FromResult(true) && + x.GetMetadataAsync(CancellationToken.None) == Task.FromResult>(null)); + } + } + } + + public abstract class AzureStorageBaseFacts + { + protected readonly Uri _baseAddress = new Uri("https://test"); + protected readonly AzureStorage _storage; + + public AzureStorageBaseFacts() + { + var directory = new Mock(); + + var client = new Mock(); + client.SetupProperty(x => x.DefaultRequestOptions); + + directory.Setup(x => x.ServiceClient).Returns(client.Object); + + _storage = new AzureStorage(directory.Object, + _baseAddress, + AzureStorage.DefaultMaxExecutionTime, + AzureStorage.DefaultServerTimeout, + false); + } + } +} \ No newline at end of file diff --git a/tests/CatalogMetadataTests/CatalogMetadataTests.csproj b/tests/CatalogMetadataTests/CatalogMetadataTests.csproj new file mode 100644 index 000000000..849056cd5 --- /dev/null +++ b/tests/CatalogMetadataTests/CatalogMetadataTests.csproj @@ -0,0 +1,78 @@ + + + + + Debug + AnyCPU + {34AABA7F-1FF7-4F4B-B1DB-D07AD4505DA4} + Library + Properties + CatalogMetadataTests + CatalogMetadataTests + v4.7.2 + 512 + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + true + true + + + + + + + + + + + + + + + + + + {e97f23b8-ecb0-4afa-b00c-015c39395fef} + NuGet.Services.Metadata.Catalog + + + {1745a383-d0be-484b-81eb-27b20f6ac6c5} + NuGet.Services.Metadata.Catalog.Monitoring + + + + + + + + 4.10.1 + + + 2.4.1 + + + 2.4.1 + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + \ No newline at end of file diff --git a/tests/CatalogMetadataTests/CommonLoggerFacts.cs b/tests/CatalogMetadataTests/CommonLoggerFacts.cs new file mode 100644 index 000000000..c793d08fe --- /dev/null +++ b/tests/CatalogMetadataTests/CommonLoggerFacts.cs @@ -0,0 +1,150 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using Moq; +using NuGet.Services.Metadata.Catalog.Monitoring; +using Xunit; + +namespace CatalogMetadataTests +{ + public class CommonLoggerFacts + { + private const string TestLogText = "dlvnsflvbjseirovj"; + + [Fact] + public void ThrowsForNullLogger() + { + Assert.Throws(() => new CommonLogger(null)); + } + + public static IEnumerable MethodToLevelsMap => new[] + { + new object[] { (Action)((l, t) => l.LogDebug(t)), LogLevel.Debug }, + new object[] { (Action)((l, t) => l.LogVerbose(t)), LogLevel.Information }, + new object[] { (Action)((l, t) => l.LogInformation(t)), LogLevel.Information }, + new object[] { (Action)((l, t) => l.LogInformationSummary(t)), LogLevel.Information }, + new object[] { (Action)((l, t) => l.LogMinimal(t)), LogLevel.Information }, + new object[] { (Action)((l, t) => l.LogWarning(t)), LogLevel.Warning }, + new object[] { (Action)((l, t) => l.LogError(t)), LogLevel.Error }, + }; + + [Theory] + [MemberData(nameof(MethodToLevelsMap))] + public void SpecificLogMethodConvertsLevelProperly(Action method, LogLevel expectedLogLevel) + { + ValidateLevel(method, expectedLogLevel); + } + + public static IEnumerable ErrorLevelsMap => new[] + { + new object[] { NuGet.Common.LogLevel.Debug, LogLevel.Debug }, + new object[] { NuGet.Common.LogLevel.Verbose, LogLevel.Information }, + new object[] { NuGet.Common.LogLevel.Information, LogLevel.Information }, + new object[] { NuGet.Common.LogLevel.Minimal, LogLevel.Information }, + new object[] { NuGet.Common.LogLevel.Warning, LogLevel.Warning }, + new object[] { NuGet.Common.LogLevel.Error, LogLevel.Error }, + }; + + [Theory] + [MemberData(nameof(ErrorLevelsMap))] + public void GenericLogConvertsLogLevelCorrectly(NuGet.Common.LogLevel inputLogLevel, LogLevel expectedLogLevel) + { + ValidateLevel((l, t) => l.Log(inputLogLevel, t), expectedLogLevel); + } + + [Theory] + [MemberData(nameof(ErrorLevelsMap))] + public void GenericLogAsyncConvertsLogLevelCorrectly(NuGet.Common.LogLevel logLevel, LogLevel expectedLogLevel) + { + ValidateLevel((l, t) => l.LogAsync(logLevel, t).Wait(), expectedLogLevel); + } + + private static NuGet.Common.NuGetLogCode[] LogCodesToTest => (NuGet.Common.NuGetLogCode[])Enum.GetValues(typeof(NuGet.Common.NuGetLogCode)); + + public static IEnumerable LevelLogCodesToTest = + from level in ErrorLevelsMap + from code in LogCodesToTest + select new object[] { level[0], level[1], code }; + + [Theory] + [MemberData(nameof(LevelLogCodesToTest))] + public void GenericLogPassesEventCodeThroughAndConvertsLogLevelCorrectly( + NuGet.Common.LogLevel logLevel, + LogLevel expectedLogLevel, + NuGet.Common.NuGetLogCode expectedCode) + { + ValidateLevelAndEventId( + (l, t) => l.Log(CreateLogMessage(logLevel, t, expectedCode)), + expectedLogLevel, + (int)expectedCode); + } + + [Theory] + [MemberData(nameof(LevelLogCodesToTest))] + public void GenericLogAsyncPassesEventCodeThroughAndConvertsLogLevelCorrectly( + NuGet.Common.LogLevel logLevel, + LogLevel expectedLogLevel, + NuGet.Common.NuGetLogCode expectedCode) + { + ValidateLevelAndEventId( + (l, t) => l.LogAsync(CreateLogMessage(logLevel, t, expectedCode)).Wait(), + expectedLogLevel, + (int)expectedCode); + } + + private static NuGet.Common.ILogMessage CreateLogMessage( + NuGet.Common.LogLevel logLevel, + string message, + NuGet.Common.NuGetLogCode code) + { + var messageMock = new Mock(); + messageMock + .SetupProperty(m => m.Level, logLevel) + .SetupProperty(m => m.Message, message) + .SetupProperty(m => m.Code, code); + + return messageMock.Object; + } + + private void ValidateLevel(Action logMethod, LogLevel expectedLogLevel) + { + var loggerMock = new Mock(MockBehavior.Strict); + loggerMock + .Setup(l => l.Log( + It.Is(level => level == expectedLogLevel), + It.IsAny(), + It.Is(s => s.ToString() == TestLogText), + It.IsAny(), + It.IsAny>())) + .Verifiable(); + + ValidateLogCall(logMethod, loggerMock); + } + + private void ValidateLevelAndEventId(Action logMethod, LogLevel expectedLogLevel, int expectedEventId) + { + var loggerMock = new Mock(MockBehavior.Strict); + loggerMock + .Setup(l => l.Log( + It.Is(level => level == expectedLogLevel), + It.Is(id => id.Id == expectedEventId), + It.Is(s => s.ToString() == TestLogText), + It.IsAny(), + It.IsAny>())) + .Verifiable(); + + ValidateLogCall(logMethod, loggerMock); + } + + private static void ValidateLogCall(Action logMethod, Mock loggerMock) + { + var logger = new CommonLogger(loggerMock.Object); + logMethod.Invoke(logger, TestLogText); + loggerMock.Verify(); + } + } +} diff --git a/tests/CatalogMetadataTests/Properties/AssemblyInfo.cs b/tests/CatalogMetadataTests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..c073b87ab --- /dev/null +++ b/tests/CatalogMetadataTests/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("CatalogMetadataTests")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("CatalogMetadataTests")] +[assembly: AssemblyCopyright("Copyright © 2017")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("34aaba7f-1ff7-4f4b-b1db-d07ad4505da4")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/tests/CatalogTests/BatchProcessingExceptionTests.cs b/tests/CatalogTests/BatchProcessingExceptionTests.cs new file mode 100644 index 000000000..35b5a0133 --- /dev/null +++ b/tests/CatalogTests/BatchProcessingExceptionTests.cs @@ -0,0 +1,30 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Services.Metadata.Catalog; +using Xunit; + +namespace CatalogTests +{ + public class BatchProcessingExceptionTests + { + [Fact] + public void Constructor_WhenExceptionIsNull_Throws() + { + var exception = Assert.Throws(() => new BatchProcessingException(inner: null)); + + Assert.Equal("inner", exception.ParamName); + } + + [Fact] + public void Constructor_WhenArgumentIsValid_ReturnsInstance() + { + var innerException = new Exception(); + var exception = new BatchProcessingException(innerException); + + Assert.Equal("A failure occurred while processing a catalog batch.", exception.Message); + Assert.Same(innerException, exception.InnerException); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/CatalogCommitBatchTaskTests.cs b/tests/CatalogTests/CatalogCommitBatchTaskTests.cs new file mode 100644 index 000000000..2abb8db23 --- /dev/null +++ b/tests/CatalogTests/CatalogCommitBatchTaskTests.cs @@ -0,0 +1,139 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; +using Xunit; + +namespace CatalogTests +{ + public class CatalogCommitBatchTaskTests + { + private readonly DateTime _minCommitTimeStamp = DateTime.UtcNow; + private static readonly PackageIdentity _packageIdentity = new PackageIdentity(id: "a", version: new NuGetVersion("1.0.0")); + private readonly CatalogCommitItemBatch _commitItemBatch; + private readonly CatalogCommitItemBatchTask _commitItemBatchTask; + + public CatalogCommitBatchTaskTests() + { + _commitItemBatch = CreateCatalogCommitItemBatch(_packageIdentity.Id); + _commitItemBatchTask = new CatalogCommitItemBatchTask(_commitItemBatch, Task.CompletedTask); + } + + [Fact] + public void Constructor_WhenBatchIsNull_Throws() + { + const CatalogCommitItemBatch batch = null; + + var exception = Assert.Throws( + () => new CatalogCommitItemBatchTask(batch, Task.CompletedTask)); + + Assert.Equal("batch", exception.ParamName); + } + + [Fact] + public void Constructor_WhenBatchKeyIsNull_Throws() + { + var commitItemBatch = CreateCatalogCommitItemBatch(key: null); + + var exception = Assert.Throws( + () => new CatalogCommitItemBatchTask(commitItemBatch, Task.CompletedTask)); + + Assert.Equal("batch.Key", exception.ParamName); + } + + [Fact] + public void Constructor_WhenTaskIsNull_Throws() + { + var exception = Assert.Throws( + () => new CatalogCommitItemBatchTask(_commitItemBatch, task: null)); + + Assert.Equal("task", exception.ParamName); + } + + [Fact] + public void Constructor_WhenArgumentIsValid_ReturnsInstance() + { + Assert.Same(_commitItemBatch, _commitItemBatchTask.Batch); + Assert.NotNull(_commitItemBatchTask.Task); + } + + [Fact] + public void GetHashCode_Always_ReturnsBatchKeyHashCode() + { + Assert.Equal(_commitItemBatch.Key.GetHashCode(), _commitItemBatchTask.GetHashCode()); + } + + [Fact] + public void Equals_WhenObjectIsNull_ReturnsFalse() + { + Assert.False(_commitItemBatchTask.Equals(obj: null)); + Assert.False(_commitItemBatchTask.Equals(other: null)); + } + + [Fact] + public void Equals_WhenObjectIsNotCatalogCommitBatchTask_ReturnsFalse() + { + Assert.False(_commitItemBatchTask.Equals(obj: new object())); + } + + [Fact] + public void Equals_WhenObjectIsSameInstance_ReturnsTrue() + { + Assert.True(_commitItemBatchTask.Equals(obj: _commitItemBatchTask)); + Assert.True(_commitItemBatchTask.Equals(other: _commitItemBatchTask)); + } + + [Fact] + public void Equals_WhenObjectBatchHasSameKey_ReturnsTrue() + { + var commitTimeStamp0 = DateTime.UtcNow; + var commitItem0 = TestUtility.CreateCatalogCommitItem(commitTimeStamp0, _packageIdentity); + var commitItemBatch0 = new CatalogCommitItemBatch(new[] { commitItem0 }, _packageIdentity.Id); + var commitItemBatchTask0 = new CatalogCommitItemBatchTask(commitItemBatch0, Task.CompletedTask); + var commitTimeStamp1 = commitTimeStamp0.AddMinutes(1); + var commitItem1 = TestUtility.CreateCatalogCommitItem(commitTimeStamp1, _packageIdentity); + var commitItemBatch1 = new CatalogCommitItemBatch(new[] { commitItem1 }, _packageIdentity.Id); + var commitItemBatchTask1 = new CatalogCommitItemBatchTask(commitItemBatch1, Task.CompletedTask); + + Assert.True(commitItemBatchTask0.Equals(obj: commitItemBatchTask1)); + Assert.True(commitItemBatchTask1.Equals(obj: commitItemBatchTask0)); + Assert.True(commitItemBatchTask0.Equals(other: commitItemBatchTask1)); + Assert.True(commitItemBatchTask1.Equals(other: commitItemBatchTask0)); + } + + [Fact] + public void Equals_WhenObjectBatchHasDifferentKey_ReturnsFalse() + { + var commitTimeStamp0 = DateTime.UtcNow; + var commitItem0 = TestUtility.CreateCatalogCommitItem( + commitTimeStamp0, + new PackageIdentity(id: "a", version: new NuGetVersion("1.0.0"))); + var commitItemBatch0 = new CatalogCommitItemBatch(new[] { commitItem0 }, key: "a"); + var commitItemBatchTask0 = new CatalogCommitItemBatchTask(commitItemBatch0, Task.CompletedTask); + var commitTimeStamp1 = commitTimeStamp0.AddMinutes(1); + var commitItem1 = TestUtility.CreateCatalogCommitItem( + commitTimeStamp1, + new PackageIdentity(id: "b", version: new NuGetVersion("1.0.0"))); + var commitItemBatch1 = new CatalogCommitItemBatch(new[] { commitItem1 }, key: "b"); + var commitItemBatchTask1 = new CatalogCommitItemBatchTask(commitItemBatch1, Task.CompletedTask); + + Assert.False(commitItemBatchTask0.Equals(obj: commitItemBatchTask1)); + Assert.False(commitItemBatchTask1.Equals(obj: commitItemBatchTask0)); + Assert.False(commitItemBatchTask0.Equals(other: commitItemBatchTask1)); + Assert.False(commitItemBatchTask1.Equals(other: commitItemBatchTask0)); + } + + private static CatalogCommitItemBatch CreateCatalogCommitItemBatch(string key) + { + var commitTimeStamp = DateTime.UtcNow; + var commitItem = TestUtility.CreateCatalogCommitItem(commitTimeStamp, _packageIdentity); + + return new CatalogCommitItemBatch(new[] { commitItem }, key); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/CatalogCommitItemBatchTests.cs b/tests/CatalogTests/CatalogCommitItemBatchTests.cs new file mode 100644 index 000000000..fd63fedcb --- /dev/null +++ b/tests/CatalogTests/CatalogCommitItemBatchTests.cs @@ -0,0 +1,60 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; +using Xunit; + +namespace CatalogTests +{ + public class CatalogCommitItemBatchTests + { + private static readonly PackageIdentity _packageIdentity = new PackageIdentity( + id: "a", + version: new NuGetVersion("1.0.0")); + + [Fact] + public void Constructor_WhenItemsIsNull_Throws() + { + var exception = Assert.Throws( + () => new CatalogCommitItemBatch(items: null)); + + Assert.Equal("items", exception.ParamName); + } + + [Fact] + public void Constructor_WhenItemsIsEmpty_Throws() + { + var exception = Assert.Throws( + () => new CatalogCommitItemBatch(Enumerable.Empty())); + + Assert.Equal("items", exception.ParamName); + } + + [Theory] + [InlineData(null)] + [InlineData("a")] + public void Constructor_WhenCommitsAreUnordered_OrdersCommitsInChronologicallyAscendingOrder(string key) + { + var commitTimeStamp = DateTime.UtcNow; + var commitItem0 = TestUtility.CreateCatalogCommitItem(commitTimeStamp, _packageIdentity); + var commitItem1 = TestUtility.CreateCatalogCommitItem(commitTimeStamp.AddMinutes(1), _packageIdentity); + var commitItem2 = TestUtility.CreateCatalogCommitItem(commitTimeStamp.AddMinutes(2), _packageIdentity); + + var commitItemBatch = new CatalogCommitItemBatch( + new[] { commitItem1, commitItem0, commitItem2 }, + key); + + Assert.Equal(commitTimeStamp, commitItemBatch.CommitTimeStamp.ToUniversalTime()); + Assert.Equal(3, commitItemBatch.Items.Count); + Assert.Equal(key, commitItemBatch.Key); + Assert.Same(commitItem0, commitItemBatch.Items[0]); + Assert.Same(commitItem1, commitItemBatch.Items[1]); + Assert.Same(commitItem2, commitItemBatch.Items[2]); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/CatalogCommitItemTests.cs b/tests/CatalogTests/CatalogCommitItemTests.cs new file mode 100644 index 000000000..a2e41c96f --- /dev/null +++ b/tests/CatalogTests/CatalogCommitItemTests.cs @@ -0,0 +1,104 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using Newtonsoft.Json.Linq; +using NgTests; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; +using Xunit; + +namespace CatalogTests +{ + public class CatalogCommitItemTests + { + private static readonly PackageIdentity _packageIdentity = new PackageIdentity(id: "A", version: new NuGetVersion("1.0.0")); + private readonly DateTime _now = DateTime.UtcNow; + private readonly JObject _context; + private readonly JObject _commitItem; + + public CatalogCommitItemTests() + { + _context = TestUtility.CreateCatalogContextJObject(); + _commitItem = TestUtility.CreateCatalogCommitItemJObject(_now, _packageIdentity); + } + + [Fact] + public void Create_WhenContextIsNull_Throws() + { + const JObject context = null; + + var exception = Assert.Throws(() => CatalogCommitItem.Create(context, _commitItem)); + + Assert.Equal("context", exception.ParamName); + } + + [Fact] + public void Create_WhenCommitItemIsNull_Throws() + { + var exception = Assert.Throws(() => CatalogCommitItem.Create(_context, commitItem: null)); + + Assert.Equal("commitItem", exception.ParamName); + } + + [Fact] + public void Create_WhenTypeIsEmpty_Throws() + { + _commitItem[CatalogConstants.TypeKeyword] = new JArray(); + + var exception = Assert.Throws(() => CatalogCommitItem.Create(_context, _commitItem)); + + Assert.Equal("commitItem", exception.ParamName); + Assert.StartsWith($"The value of property '{CatalogConstants.TypeKeyword}' must be non-null and non-empty.", exception.Message); + } + + [Fact] + public void Create_WhenArgumentsAreValid_ReturnsInstance() + { + var commitItem = CatalogCommitItem.Create(_context, _commitItem); + + Assert.Equal($"https://nuget.test/{_packageIdentity.Id}", commitItem.Uri.AbsoluteUri); + Assert.Equal(_now, commitItem.CommitTimeStamp.ToUniversalTime()); + Assert.True(Guid.TryParse(commitItem.CommitId, out var commitId)); + Assert.Equal(_packageIdentity, commitItem.PackageIdentity); + Assert.Equal(CatalogConstants.NuGetPackageDetails, commitItem.Types.Single()); + Assert.Equal(Schema.DataTypes.PackageDetails.AbsoluteUri, commitItem.TypeUris.Single().AbsoluteUri); + } + + [Fact] + public void CompareTo_WhenObjIsNull_Throws() + { + var commitItem = CatalogCommitItem.Create(_context, _commitItem); + + var exception = Assert.Throws(() => commitItem.CompareTo(obj: null)); + + Assert.Equal("obj", exception.ParamName); + } + + [Fact] + public void CompareTo_WhenObjIsNotCatalogCommit_Throws() + { + var commitItem = CatalogCommitItem.Create(_context, _commitItem); + + var exception = Assert.Throws(() => commitItem.CompareTo(new object())); + + Assert.Equal("obj", exception.ParamName); + } + + [Fact] + public void CompareTo_WhenArgumentIsValid_ReturnsValue() + { + var commitTimeStamp1 = DateTime.UtcNow; + var commitTimeStamp2 = DateTime.UtcNow.AddMinutes(1); + var commitItem0 = TestUtility.CreateCatalogCommitItem(commitTimeStamp1, _packageIdentity); + var commitItem1 = TestUtility.CreateCatalogCommitItem(commitTimeStamp2, _packageIdentity); + + Assert.Equal(0, commitItem0.CompareTo(commitItem0)); + Assert.Equal(-1, commitItem0.CompareTo(commitItem1)); + Assert.Equal(1, commitItem1.CompareTo(commitItem0)); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/CatalogCommitTests.cs b/tests/CatalogTests/CatalogCommitTests.cs new file mode 100644 index 000000000..8b8439507 --- /dev/null +++ b/tests/CatalogTests/CatalogCommitTests.cs @@ -0,0 +1,85 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json.Linq; +using NgTests; +using NuGet.Services.Metadata.Catalog; +using Xunit; + +namespace CatalogTests +{ + public class CatalogCommitTests + { + [Fact] + public void Create_WhenCommitIsNull_Throws() + { + var exception = Assert.Throws(() => CatalogCommit.Create(commit: null)); + + Assert.Equal("commit", exception.ParamName); + } + + [Fact] + public void Create_WhenArgumentIsValid_ReturnsInstance() + { + var idKeyword = "https://nuget.test/a"; + var commitTimeStamp = DateTime.UtcNow.ToString("O"); + var jObject = new JObject( + new JProperty(CatalogConstants.IdKeyword, idKeyword), + new JProperty(CatalogConstants.CommitTimeStamp, commitTimeStamp)); + + var commit = CatalogCommit.Create(jObject); + + Assert.Equal(idKeyword, commit.Uri.AbsoluteUri); + Assert.Equal(commitTimeStamp, commit.CommitTimeStamp.ToUniversalTime().ToString("O")); + } + + [Fact] + public void CompareTo_WhenObjIsNull_Throws() + { + var commit = CreateCatalogCommit(); + + var exception = Assert.Throws(() => commit.CompareTo(obj: null)); + + Assert.Equal("obj", exception.ParamName); + } + + [Fact] + public void CompareTo_WhenObjIsNotCatalogCommit_Throws() + { + var commit = CreateCatalogCommit(); + + var exception = Assert.Throws(() => commit.CompareTo(new object())); + + Assert.Equal("obj", exception.ParamName); + } + + [Fact] + public void CompareTo_WhenObjIsCatalogCommit_ReturnsValue() + { + var jObject0 = new JObject( + new JProperty(CatalogConstants.IdKeyword, "https://nuget.test/a"), + new JProperty(CatalogConstants.CommitTimeStamp, DateTime.UtcNow.ToString("O"))); + + var jObject1 = new JObject( + new JProperty(CatalogConstants.IdKeyword, "https://nuget.test/b"), + new JProperty(CatalogConstants.CommitTimeStamp, DateTime.UtcNow.AddHours(1).ToString("O"))); + + var commit0 = CatalogCommit.Create(jObject0); + var commit1 = CatalogCommit.Create(jObject1); + + Assert.Equal(-1, commit0.CompareTo(commit1)); + Assert.Equal(0, commit0.CompareTo(commit0)); + Assert.Equal(1, commit1.CompareTo(commit0)); + } + + private static CatalogCommit CreateCatalogCommit() + { + var jObject = new JObject( + new JProperty(CatalogConstants.IdKeyword, "https://nuget.test/a"), + new JProperty(CatalogConstants.CommitTimeStamp, DateTime.UtcNow.ToString("O"))); + + return CatalogCommit.Create(jObject); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/CatalogCommitUtilitiesTests.cs b/tests/CatalogTests/CatalogCommitUtilitiesTests.cs new file mode 100644 index 000000000..41a75efe4 --- /dev/null +++ b/tests/CatalogTests/CatalogCommitUtilitiesTests.cs @@ -0,0 +1,479 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; +using Xunit; + +namespace CatalogTests +{ + public class CatalogCommitUtilitiesTests + { + private const int _maxConcurrentBatches = 1; + private static readonly CatalogCommitItemBatch _lastBatch; + private static readonly Task FailedTask = Task.FromException(new Exception()); + private static readonly PackageIdentity _packageIdentitya = new PackageIdentity(id: "a", version: new NuGetVersion("1.0.0")); + private static readonly PackageIdentity _packageIdentityb = new PackageIdentity(id: "b", version: new NuGetVersion("1.0.0")); + + static CatalogCommitUtilitiesTests() + { + var commitTimeStamp = DateTime.UtcNow; + var commit = TestUtility.CreateCatalogCommitItem(commitTimeStamp, _packageIdentitya); + + _lastBatch = new CatalogCommitItemBatch(new[] { commit }, _packageIdentitya.Id); + } + + [Fact] + public void CreateCommitItemBatches_WhenCatalogItemsIsNull_Throws() + { + IEnumerable catalogItems = null; + + var exception = Assert.Throws( + () => CatalogCommitUtilities.CreateCommitItemBatches( + catalogItems, + CatalogCommitUtilities.GetPackageIdKey)); + + Assert.Equal("catalogItems", exception.ParamName); + } + + [Fact] + public void CreateCommitItemBatches_WhenGetCatalogCommitItemKeyIsNull_Throws() + { + var exception = Assert.Throws( + () => CatalogCommitUtilities.CreateCommitItemBatches( + Enumerable.Empty(), + getCatalogCommitItemKey: null)); + + Assert.Equal("getCatalogCommitItemKey", exception.ParamName); + } + + [Fact] + public void CreateCommitItemBatches_WhenMultipleCommitItemsShareCommitTimeStampButNotCommitId_Throws() + { + var commitTimeStamp = DateTime.UtcNow; + var context = TestUtility.CreateCatalogContextJObject(); + var commitItem0 = CatalogCommitItem.Create( + context, + TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp, _packageIdentitya)); + var commitItem1 = CatalogCommitItem.Create( + context, + TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp, _packageIdentityb)); + var commitItems = new[] { commitItem0, commitItem1 }; + + var exception = Assert.Throws( + () => CatalogCommitUtilities.CreateCommitItemBatches( + commitItems, + CatalogCommitUtilities.GetPackageIdKey)); + + Assert.Equal("catalogItems", exception.ParamName); + Assert.StartsWith("Multiple commits exist with the same commit timestamp but different commit ID's: " + + $"{{ CommitId = {commitItem0.CommitId}, CommitTimeStamp = {commitItem0.CommitTimeStamp.ToString("O")} }}, " + + $"{{ CommitId = {commitItem1.CommitId}, CommitTimeStamp = {commitItem1.CommitTimeStamp.ToString("O")} }}.", + exception.Message); + } + + [Fact] + public void CreateCommitItemBatches_WhenMultipleCommitItemsShareCommitTimeStampButNotCommitIdAndLaterCommitExists_DoesNotThrow() + { + var commitTimeStamp0 = DateTime.UtcNow; + var commitTimeStamp1 = commitTimeStamp0.AddMinutes(1); + var context = TestUtility.CreateCatalogContextJObject(); + var commitItem0 = CatalogCommitItem.Create( + context, + TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp0, _packageIdentitya)); + var commitItem1 = CatalogCommitItem.Create( + context, + TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp0, _packageIdentityb)); + var commitItem2 = CatalogCommitItem.Create( + context, + TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp1, _packageIdentitya)); + var commitItems = new[] { commitItem0, commitItem1, commitItem2 }; + + var batches = CatalogCommitUtilities.CreateCommitItemBatches( + commitItems, + CatalogCommitUtilities.GetPackageIdKey); + + Assert.Collection( + batches, + batch => + { + Assert.Equal(commitTimeStamp1, batch.CommitTimeStamp.ToUniversalTime()); + Assert.Collection( + batch.Items, + commit => Assert.True(ReferenceEquals(commit, commitItem2))); + }, + batch => + { + Assert.Equal(commitTimeStamp0, batch.CommitTimeStamp.ToUniversalTime()); + Assert.Collection( + batch.Items, + commit => Assert.True(ReferenceEquals(commit, commitItem1))); + }); + } + + [Fact] + public void CreateCommitItemBatches_WhenPackageIdsVaryInCasing_GroupsUsingProvidedGetKeyFunction() + { + var now = DateTime.UtcNow; + var packageIdentityA0 = new PackageIdentity(id: "a", version: new NuGetVersion("1.0.0")); + var packageIdentityA1 = new PackageIdentity(id: "A", version: new NuGetVersion("2.0.0")); + var packageIdentityA2 = new PackageIdentity(id: "A", version: new NuGetVersion("3.0.0")); + var packageIdentityB0 = new PackageIdentity(id: "b", version: new NuGetVersion("1.0.0")); + var packageIdentityB1 = new PackageIdentity(id: "B", version: new NuGetVersion("2.0.0")); + var commitId0 = Guid.NewGuid().ToString(); + var commitId1 = Guid.NewGuid().ToString(); + var commitId2 = Guid.NewGuid().ToString(); + var commitItem0 = TestUtility.CreateCatalogCommitItem(now, packageIdentityA0, commitId0); + var commitItem1 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(1), packageIdentityA1, commitId1); + var commitItem2 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(2), packageIdentityA2, commitId2); + var commitItem3 = TestUtility.CreateCatalogCommitItem(now, packageIdentityB0, commitId0); + var commitItem4 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(1), packageIdentityB1, commitId1); + + // not in alphanumeric or chronological order + var commitItems = new[] { commitItem4, commitItem2, commitItem0, commitItem3, commitItem1 }; + + var batches = CatalogCommitUtilities.CreateCommitItemBatches( + commitItems, + CatalogCommitUtilities.GetPackageIdKey); + + Assert.Collection( + batches, + batch => + { + Assert.Equal(commitItem3.CommitTimeStamp, batch.CommitTimeStamp); + Assert.Collection( + batch.Items, + commit => Assert.True(ReferenceEquals(commit, commitItem3)), + commit => Assert.True(ReferenceEquals(commit, commitItem4))); + }, + batch => + { + Assert.Equal(commitItem0.CommitTimeStamp, batch.CommitTimeStamp); + Assert.Collection( + batch.Items, + commit => Assert.True(ReferenceEquals(commit, commitItem0)), + commit => Assert.True(ReferenceEquals(commit, commitItem1)), + commit => Assert.True(ReferenceEquals(commit, commitItem2))); + }); + } + + [Fact] + public void CreateCommitItemBatches_WhenCommitItemsContainMultipleCommitsForSamePackageIdentity_ReturnsOnlyLastCommitForEachPackageIdentity() + { + var now = DateTime.UtcNow; + var commitItem0 = TestUtility.CreateCatalogCommitItem(now, _packageIdentitya); + var commitItem1 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(1), _packageIdentitya); + var commitItem2 = TestUtility.CreateCatalogCommitItem(now.AddMinutes(2), _packageIdentitya); + var commitItems = new[] { commitItem0, commitItem1, commitItem2 }; + + var batches = CatalogCommitUtilities.CreateCommitItemBatches(commitItems, CatalogCommitUtilities.GetPackageIdKey); + + Assert.Collection( + batches, + batch => + { + Assert.Equal(commitItem2.CommitTimeStamp, batch.CommitTimeStamp); + Assert.Collection( + batch.Items, + commit => Assert.True(ReferenceEquals(commit, commitItem2))); + }); + } + + [Fact] + public void StartProcessingBatchesIfNoFailures_WhenClientIsNull_Throws() + { + const CollectorHttpClient client = null; + + var exception = Assert.Throws( + () => CatalogCommitUtilities.StartProcessingBatchesIfNoFailures( + client, + new JObject(), + new List(), + new List(), + _maxConcurrentBatches, + NoOpProcessBatchAsync, + CancellationToken.None)); + + Assert.Equal("client", exception.ParamName); + } + + [Fact] + public void StartProcessingBatchesIfNoFailures_WhenContextIsNull_Throws() + { + const JToken context = null; + + var exception = Assert.Throws( + () => CatalogCommitUtilities.StartProcessingBatchesIfNoFailures( + new CollectorHttpClient(), + context, + new List(), + new List(), + _maxConcurrentBatches, + NoOpProcessBatchAsync, + CancellationToken.None)); + + Assert.Equal("context", exception.ParamName); + } + + [Fact] + public void StartProcessingBatchesIfNoFailures_WhenUnprocessedBatchesIsNull_Throws() + { + const List unprocessedBatches = null; + + var exception = Assert.Throws( + () => CatalogCommitUtilities.StartProcessingBatchesIfNoFailures( + new CollectorHttpClient(), + new JObject(), + unprocessedBatches, + new List(), + _maxConcurrentBatches, + NoOpProcessBatchAsync, + CancellationToken.None)); + + Assert.Equal("unprocessedBatches", exception.ParamName); + } + + [Fact] + public void StartProcessingBatchesIfNoFailures_WhenProcessingBatchesIsNull_Throws() + { + const List processingBatches = null; + + var exception = Assert.Throws( + () => CatalogCommitUtilities.StartProcessingBatchesIfNoFailures( + new CollectorHttpClient(), + new JObject(), + new List(), + processingBatches, + _maxConcurrentBatches, + NoOpProcessBatchAsync, + CancellationToken.None)); + + Assert.Equal("processingBatches", exception.ParamName); + } + + [Fact] + public void StartProcessingBatchesIfNoFailures_WhenMaxConcurrentBatchesIsLessThanOne_Throws() + { + const int maxConcurrentBatches = 0; + + var exception = Assert.Throws( + () => CatalogCommitUtilities.StartProcessingBatchesIfNoFailures( + new CollectorHttpClient(), + new JObject(), + new List(), + new List(), + maxConcurrentBatches, + NoOpProcessBatchAsync, + CancellationToken.None)); + + Assert.Equal("maxConcurrentBatches", exception.ParamName); + } + + [Fact] + public void StartProcessingBatchesIfNoFailures_WhenProcessCommitItemBatchAsyncIsNull_Throws() + { + const ProcessCommitItemBatchAsync processCommitItemBatchAsync = null; + + var exception = Assert.Throws( + () => CatalogCommitUtilities.StartProcessingBatchesIfNoFailures( + new CollectorHttpClient(), + new JObject(), + new List(), + new List(), + _maxConcurrentBatches, + processCommitItemBatchAsync, + CancellationToken.None)); + + Assert.Equal("processCommitItemBatchAsync", exception.ParamName); + } + + public class StartProcessingBatchesIfNoFailures + { + private PackageIdentity _packageIdentityc = new PackageIdentity(id: "c", version: new NuGetVersion("1.0.0")); + private PackageIdentity _packageIdentityd = new PackageIdentity(id: "d", version: new NuGetVersion("1.0.0")); + private readonly DateTime _now = DateTime.UtcNow; + private readonly CatalogCommitItem _commitItem0; + private readonly CatalogCommitItem _commitItem1; + private readonly CatalogCommitItem _commitItem2; + private readonly CatalogCommitItem _commitItem3; + + public StartProcessingBatchesIfNoFailures() + { + _commitItem0 = TestUtility.CreateCatalogCommitItem(_now, _packageIdentitya); + _commitItem1 = TestUtility.CreateCatalogCommitItem(_now, _packageIdentityb); + _commitItem2 = TestUtility.CreateCatalogCommitItem(_now.AddMinutes(1), _packageIdentityc); + _commitItem3 = TestUtility.CreateCatalogCommitItem(_now.AddMinutes(2), _packageIdentityd); + } + + [Fact] + public void StartProcessingBatchesIfNoFailures_WhenAnyBatchIsFailed_DoesNotStartNewBatch() + { + var commitItemBatch = new CatalogCommitItemBatch( + new[] { _commitItem0, _commitItem1 }, + _packageIdentitya.Id); + var failedBatchTask = new CatalogCommitItemBatchTask(commitItemBatch, FailedTask); + var unprocessedBatches = new List() { commitItemBatch }; + var processingBatches = new List() { failedBatchTask }; + + CatalogCommitUtilities.StartProcessingBatchesIfNoFailures( + new CollectorHttpClient(), + new JObject(), + unprocessedBatches, + processingBatches, + _maxConcurrentBatches, + NoOpProcessBatchAsync, + CancellationToken.None); + + Assert.Single(unprocessedBatches); + Assert.Single(processingBatches); + } + + [Fact] + public void StartProcessingBatchesIfNoFailures_WhenNoBatchIsCancelled_DoesNotStartNewBatch() + { + var commitItemBatch = new CatalogCommitItemBatch( + new[] { _commitItem0, _commitItem1 }, + _packageIdentitya.Id); + var cancelledBatchTask = new CatalogCommitItemBatchTask( + commitItemBatch, + Task.FromCanceled(new CancellationToken(canceled: true))); + var unprocessedBatches = new List() { commitItemBatch }; + var processingBatches = new List() { cancelledBatchTask }; + + CatalogCommitUtilities.StartProcessingBatchesIfNoFailures( + new CollectorHttpClient(), + new JObject(), + unprocessedBatches, + processingBatches, + _maxConcurrentBatches, + NoOpProcessBatchAsync, + CancellationToken.None); + + Assert.Single(unprocessedBatches); + Assert.Single(processingBatches); + } + + [Fact] + public void StartProcessingBatchesIfNoFailures_WhenMaxConcurrencyLimitHit_DoesNotStartNewBatch() + { + var commitItemBatch0 = new CatalogCommitItemBatch(new[] { _commitItem0 }, _packageIdentitya.Id); + var commitItemBatch1 = new CatalogCommitItemBatch(new[] { _commitItem2 }, _packageIdentityc.Id); + + using (var cancellationTokenSource = new CancellationTokenSource()) + { + var inProcessTask = new CatalogCommitItemBatchTask( + commitItemBatch0, + Task.Delay(TimeSpan.FromMilliseconds(-1), cancellationTokenSource.Token)); + var unprocessedBatches = new List() { commitItemBatch1 }; + var processingBatches = new List() { inProcessTask }; + + const int maxConcurrentBatches = 1; + + CatalogCommitUtilities.StartProcessingBatchesIfNoFailures( + new CollectorHttpClient(), + new JObject(), + unprocessedBatches, + processingBatches, + maxConcurrentBatches, + NoOpProcessBatchAsync, + CancellationToken.None); + + Assert.Single(unprocessedBatches); + Assert.Single(processingBatches); + } + } + + [Fact] + public void StartProcessingBatchesIfNoFailures_WhenCanStartNewBatch_StartsNewBatch() + { + var commitItemBatch = new CatalogCommitItemBatch(new[] { _commitItem0 }, _packageIdentitya.Id); + var unprocessedBatches = new List() { commitItemBatch }; + var processingBatches = new List(); + + CatalogCommitUtilities.StartProcessingBatchesIfNoFailures( + new CollectorHttpClient(), + new JObject(), + unprocessedBatches, + processingBatches, + _maxConcurrentBatches, + NoOpProcessBatchAsync, + CancellationToken.None); + + Assert.Empty(unprocessedBatches); + Assert.Single(processingBatches); + } + + [Fact] + public void StartProcessingBatchesIfNoFailures_WhenProcessingQueueContainsCompletedTasks_StartsNewBatch() + { + var commitItemBatch0 = new CatalogCommitItemBatch(new[] { _commitItem0 }, _packageIdentitya.Id); + var commitItemBatch1 = new CatalogCommitItemBatch(new[] { _commitItem1 }, _packageIdentityb.Id); + var completedTask = new CatalogCommitItemBatchTask(commitItemBatch0, Task.CompletedTask); + var unprocessedBatches = new List() { commitItemBatch1 }; + var processingBatches = new List() { completedTask }; + + CatalogCommitUtilities.StartProcessingBatchesIfNoFailures( + new CollectorHttpClient(), + new JObject(), + unprocessedBatches, + processingBatches, + _maxConcurrentBatches, + NoOpProcessBatchAsync, + CancellationToken.None); + + Assert.Empty(unprocessedBatches); + Assert.Equal(2, processingBatches.Count); + } + } + + [Fact] + public void GetPackageIdKey_WhenItemIsNull_Throws() + { + var exception = Assert.Throws( + () => CatalogCommitUtilities.GetPackageIdKey(item: null)); + + Assert.Equal("item", exception.ParamName); + } + + [Theory] + [InlineData("a")] + [InlineData("A")] + public void GetPackageIdKey_WhenPackageIdVariesInCase_ReturnsLowerCase(string packageId) + { + var commitItem = TestUtility.CreateCatalogCommitItem( + DateTime.UtcNow, + new PackageIdentity(packageId, new NuGetVersion("1.0.0"))); + var key = CatalogCommitUtilities.GetPackageIdKey(commitItem); + + Assert.Equal(packageId.ToLowerInvariant(), key); + } + + private static CatalogCommitItemBatch CreateCatalogCommitBatch( + DateTime commitTimeStamp, + PackageIdentity packageIdentity) + { + var commit = TestUtility.CreateCatalogCommitItem(commitTimeStamp, packageIdentity); + + return new CatalogCommitItemBatch(new[] { commit }, packageIdentity.Id); + } + + private static Task NoOpProcessBatchAsync( + CollectorHttpClient client, + JToken context, + string packageId, + CatalogCommitItemBatch batch, + CatalogCommitItemBatch lastBatch, + CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/CatalogIndexEntryTests.cs b/tests/CatalogTests/CatalogIndexEntryTests.cs new file mode 100644 index 000000000..224098bf1 --- /dev/null +++ b/tests/CatalogTests/CatalogIndexEntryTests.cs @@ -0,0 +1,351 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Linq; +using NgTests; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Protocol; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; +using Xunit; + +namespace CatalogTests +{ + public class CatalogIndexEntryTests + { + private static readonly JsonSerializerSettings _settings; + + private readonly string _commitId; + private readonly DateTime _commitTimeStamp; + private const string _packageId = "a"; + private readonly NuGetVersion _packageVersion; + private readonly PackageIdentity _packageIdentity; + private readonly Uri _uri; + + static CatalogIndexEntryTests() + { + _settings = new JsonSerializerSettings(); + + _settings.Converters.Add(new NuGetVersionConverter()); + _settings.Converters.Add(new StringEnumConverter()); + } + + public CatalogIndexEntryTests() + { + _commitId = Guid.NewGuid().ToString(); + _commitTimeStamp = DateTime.UtcNow; + _packageVersion = new NuGetVersion("1.2.3"); + _uri = new Uri("https://nuget.test/a"); + _packageIdentity = new PackageIdentity(_packageId, _packageVersion); + } + + [Fact] + public void Constructor_WhenUriIsNull_Throws() + { + const Uri uri = null; + + var exception = Assert.Throws( + () => new CatalogIndexEntry( + uri, + CatalogConstants.NuGetPackageDetails, + _commitId, + _commitTimeStamp, + _packageIdentity)); + + Assert.Equal("uri", exception.ParamName); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void Constructor_WhenTypeIsNullEmptyOrWhitespace_Throws(string type) + { + var exception = Assert.Throws( + () => new CatalogIndexEntry( + _uri, + type, + _commitId, + _commitTimeStamp, + _packageIdentity)); + + Assert.Equal("type", exception.ParamName); + } + + [Fact] + public void Constructor_WhenTypesIsNull_Throws() + { + string[] types = null; + + var exception = Assert.Throws( + () => new CatalogIndexEntry( + _uri, + types, + _commitId, + _commitTimeStamp, + _packageIdentity)); + + Assert.Equal("types", exception.ParamName); + } + + [Fact] + public void Constructor_WhenTypesIsEmpty_Throws() + { + var exception = Assert.Throws( + () => new CatalogIndexEntry( + _uri, + Array.Empty(), + _commitId, + _commitTimeStamp, + _packageIdentity)); + + Assert.Equal("types", exception.ParamName); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void Constructor_WhenCommitIdIsNullEmptyOrWhitespace_Throws(string commitId) + { + var exception = Assert.Throws( + () => new CatalogIndexEntry( + _uri, + CatalogConstants.NuGetPackageDelete, + commitId, + _commitTimeStamp, + _packageIdentity)); + + Assert.Equal("commitId", exception.ParamName); + } + + [Fact] + public void Constructor_WhenPackageIdentityIsNull_Throws() + { + var exception = Assert.Throws( + () => new CatalogIndexEntry( + _uri, + CatalogConstants.NuGetPackageDelete, + _commitId, + _commitTimeStamp, + packageIdentity: null)); + + Assert.Equal("packageIdentity", exception.ParamName); + } + + [Fact] + public void Constructor_WhenArgumentsAreValid_ReturnsInstance() + { + var entry = new CatalogIndexEntry( + _uri, + CatalogConstants.NuGetPackageDetails, + _commitId, + _commitTimeStamp, + _packageIdentity); + + Assert.Equal(_uri.AbsoluteUri, entry.Uri.AbsoluteUri); + Assert.Equal(CatalogConstants.NuGetPackageDetails, entry.Types.Single()); + Assert.Equal(_commitId, entry.CommitId); + Assert.Equal(_commitTimeStamp, entry.CommitTimeStamp); + Assert.Equal(_packageId, entry.Id); + Assert.Equal(_packageVersion, entry.Version); + } + + [Fact] + public void Create_WhenCommitItemIsNull_Throws() + { + var exception = Assert.Throws(() => CatalogIndexEntry.Create(commitItem: null)); + + Assert.Equal("commitItem", exception.ParamName); + } + + [Fact] + public void Create_WhenArgumentIsValid_ReturnsInstance() + { + var contextJObject = TestUtility.CreateCatalogContextJObject(); + var commitItemJObject = TestUtility.CreateCatalogCommitItemJObject(_commitTimeStamp, _packageIdentity); + var commitItem = CatalogCommitItem.Create(contextJObject, commitItemJObject); + + var entry = CatalogIndexEntry.Create(commitItem); + + Assert.Equal(_uri.AbsoluteUri, entry.Uri.AbsoluteUri); + Assert.Equal(CatalogConstants.NuGetPackageDetails, entry.Types.Single()); + Assert.Equal(commitItemJObject[CatalogConstants.CommitId].ToString(), entry.CommitId); + Assert.Equal(_commitTimeStamp, entry.CommitTimeStamp.ToUniversalTime()); + Assert.Equal(_packageId, entry.Id); + Assert.Equal(_packageVersion, entry.Version); + } + + [Fact] + public void CompareTo_WhenOtherIsNull_Throws() + { + var entry = new CatalogIndexEntry( + _uri, + CatalogConstants.NuGetPackageDetails, + _commitId, + _commitTimeStamp, + _packageIdentity); + + var exception = Assert.Throws(() => entry.CompareTo(other: null)); + + Assert.Equal("other", exception.ParamName); + } + + [Fact] + public void CompareTo_WhenCommitTimeStampsAreNotEqual_ReturnsNonZero() + { + var now = DateTime.UtcNow; + var olderEntry = new CatalogIndexEntry( + _uri, + CatalogConstants.NuGetPackageDetails, + _commitId, + now.AddHours(-1), + _packageIdentity); + var newerEntry = new CatalogIndexEntry( + _uri, + CatalogConstants.NuGetPackageDetails, + _commitId, + now, + _packageIdentity); + + Assert.Equal(-1, olderEntry.CompareTo(newerEntry)); + Assert.Equal(1, newerEntry.CompareTo(olderEntry)); + } + + [Fact] + public void CompareTo_WhenCommitTimeStampsAreEqual_ReturnsZero() + { + var entry0 = new CatalogIndexEntry( + new Uri("https://nuget.test/a"), + CatalogConstants.NuGetPackageDetails, + _commitId, + _commitTimeStamp, + _packageIdentity); + var entry1 = new CatalogIndexEntry( + new Uri("https://nuget.test/b"), + CatalogConstants.NuGetPackageDelete, + Guid.NewGuid().ToString(), + _commitTimeStamp, + new PackageIdentity(id: "b", version: new NuGetVersion("4.5.6"))); + + Assert.Equal(0, entry0.CompareTo(entry0)); + Assert.Equal(0, entry0.CompareTo(entry1)); + Assert.Equal(0, entry1.CompareTo(entry0)); + } + + [Fact] + public void IsDelete_WhenTypeIsNotPackageDelete_ReturnsFalse() + { + var entry = new CatalogIndexEntry( + _uri, + CatalogConstants.NuGetPackageDetails, + _commitId, + _commitTimeStamp, + _packageIdentity); + + Assert.False(entry.IsDelete); + } + + [Fact] + public void IsDelete_WhenTypeIsPackageDelete_ReturnsTrue() + { + var entry = new CatalogIndexEntry( + _uri, + CatalogConstants.NuGetPackageDelete, + _commitId, + _commitTimeStamp, + _packageIdentity); + + Assert.True(entry.IsDelete); + } + + [Fact] + public void JsonSerialization_ReturnsCorrectJson() + { + var entry = new CatalogIndexEntry( + _uri, + CatalogConstants.NuGetPackageDetails, + _commitId, + _commitTimeStamp, + _packageIdentity); + + var jObject = CreateCatalogIndexJObject(CatalogConstants.NuGetPackageDetails); + + var expectedResult = jObject.ToString(Formatting.None, _settings.Converters.ToArray()); + var actualResult = JsonConvert.SerializeObject(entry, Formatting.None, _settings); + + Assert.Equal(expectedResult, actualResult); + } + + [Theory] + [InlineData(CatalogConstants.IdKeyword)] + [InlineData(CatalogConstants.TypeKeyword)] + [InlineData(CatalogConstants.CommitId)] + [InlineData(CatalogConstants.CommitTimeStamp)] + [InlineData(CatalogConstants.NuGetId)] + [InlineData(CatalogConstants.NuGetVersion)] + public void JsonDeserialization_WhenRequiredPropertyIsMissing_Throws(string propertyToRemove) + { + var jObject = CreateCatalogIndexJObject(CatalogConstants.NuGetPackageDetails); + + jObject.Remove(propertyToRemove); + + var json = jObject.ToString(Formatting.None, _settings.Converters.ToArray()); + + var exception = Assert.Throws( + () => JsonConvert.DeserializeObject(json, _settings)); + + Assert.StartsWith($"Required property '{propertyToRemove}' not found in JSON.", exception.Message); + } + + [Fact] + public void JsonDeserialization_WhenTypeIsPackageDetails_ReturnsCorrectObject() + { + var jObject = CreateCatalogIndexJObject(CatalogConstants.NuGetPackageDetails); + var json = jObject.ToString(Formatting.None, _settings.Converters.ToArray()); + + var entry = JsonConvert.DeserializeObject(json, _settings); + + Assert.Equal(_uri.AbsoluteUri, entry.Uri.AbsoluteUri); + Assert.Equal(CatalogConstants.NuGetPackageDetails, entry.Types.Single()); + Assert.False(entry.IsDelete); + Assert.Equal(_commitId, entry.CommitId); + Assert.Equal(_commitTimeStamp, entry.CommitTimeStamp); + Assert.Equal(_packageId, entry.Id); + Assert.Equal(_packageVersion, entry.Version); + } + + [Fact] + public void JsonDeserialization_WhenTypeIsPackageDelete_ReturnsCorrectObject() + { + var jObject = CreateCatalogIndexJObject(CatalogConstants.NuGetPackageDelete); + var json = jObject.ToString(Formatting.None, _settings.Converters.ToArray()); + + var entry = JsonConvert.DeserializeObject(json, _settings); + + Assert.Equal(_uri.AbsoluteUri, entry.Uri.AbsoluteUri); + Assert.Equal(CatalogConstants.NuGetPackageDelete, entry.Types.Single()); + Assert.True(entry.IsDelete); + Assert.Equal(_commitId, entry.CommitId); + Assert.Equal(_commitTimeStamp, entry.CommitTimeStamp); + Assert.Equal(_packageId, entry.Id); + Assert.Equal(_packageVersion, entry.Version); + } + + private JObject CreateCatalogIndexJObject(string type) + { + return new JObject( + new JProperty(CatalogConstants.IdKeyword, _uri), + new JProperty(CatalogConstants.TypeKeyword, type), + new JProperty(CatalogConstants.CommitId, _commitId), + new JProperty(CatalogConstants.CommitTimeStamp, _commitTimeStamp.ToString(CatalogConstants.CommitTimeStampFormat)), + new JProperty(CatalogConstants.NuGetId, _packageId), + new JProperty(CatalogConstants.NuGetVersion, _packageVersion.ToNormalizedString())); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/CatalogIndexReaderTests.cs b/tests/CatalogTests/CatalogIndexReaderTests.cs new file mode 100644 index 000000000..c182405cb --- /dev/null +++ b/tests/CatalogTests/CatalogIndexReaderTests.cs @@ -0,0 +1,69 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Moq; +using NgTests.Data; +using NgTests.Infrastructure; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; +using Xunit; + +namespace CatalogTests +{ + public class CatalogIndexReaderTests + { + [Fact] + public async Task GetEntries() + { + // Arrange + var indexUri = "http://tempuri.org/index.json"; + var responses = new Dictionary() + { + { "http://tempuri.org/index.json", TestCatalogEntries.TestCatalogStorageWithThreePackagesIndex }, + { "http://tempuri.org/page0.json", TestCatalogEntries.TestCatalogStorageWithThreePackagesPage }, + }; + + var reader = new CatalogIndexReader( + new Uri(indexUri), + new CollectorHttpClient(new InMemoryHttpHandler(responses)), + new Mock().Object); + + // Act + var entries = await reader.GetEntries(); + + // Assert + var entryList = entries.ToList(); + Assert.Equal(3, entryList.Count); + + Assert.Equal("http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json", entryList[0].Uri.ToString()); + Assert.Equal("http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json", entryList[1].Uri.ToString()); + Assert.Equal("http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json", entryList[2].Uri.ToString()); + + Assert.Equal("2015-10-12T10:08:55.3335317", entryList[0].CommitTimeStamp.ToString("O")); + Assert.Equal("2015-10-12T10:08:54.1506742", entryList[1].CommitTimeStamp.ToString("O")); + Assert.Equal("2015-10-12T10:08:54.1506742", entryList[2].CommitTimeStamp.ToString("O")); + + Assert.Equal("8a9e7694-73d4-4775-9b7a-20aa59b9773e", entryList[0].CommitId); + Assert.Equal("9a37734f-1960-4c07-8934-c8bc797e35c1", entryList[1].CommitId); + Assert.Equal("9a37734f-1960-4c07-8934-c8bc797e35c1", entryList[2].CommitId); + + Assert.Equal("ListedPackage", entryList[0].Id); + Assert.Equal("ListedPackage", entryList[1].Id); + Assert.Equal("UnlistedPackage", entryList[2].Id); + + Assert.Equal(new NuGetVersion("1.0.1"), entryList[0].Version); + Assert.Equal(new NuGetVersion("1.0.0"), entryList[1].Version); + Assert.Equal(new NuGetVersion("1.0.0"), entryList[2].Version); + + Assert.Equal(new[] { "nuget:PackageDetails" }, entryList[0].Types); + Assert.Equal(new[] { "nuget:PackageDetails" }, entryList[1].Types); + Assert.Equal(new[] { "nuget:PackageDetails" }, entryList[2].Types); + + Assert.Same(entryList[0].Id, entryList[1].Id); + } + } +} diff --git a/tests/CatalogTests/CatalogTests.csproj b/tests/CatalogTests/CatalogTests.csproj new file mode 100644 index 000000000..010285e2d --- /dev/null +++ b/tests/CatalogTests/CatalogTests.csproj @@ -0,0 +1,359 @@ + + + + + Debug + AnyCPU + {4D0B6BAB-5A33-4A7F-B007-93194FC2E2E3} + Library + Properties + CatalogTests + CatalogTests + v4.7.2 + 512 + + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + true + bin\x64\Debug\ + DEBUG;TRACE + full + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + bin\x64\Release\ + TRACE + true + pdbonly + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + + + + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + {e97f23b8-ecb0-4afa-b00c-015c39395fef} + NuGet.Services.Metadata.Catalog + + + {d44c2e89-2d98-44bd-8712-8ccbe4e67c9c} + NuGet.Protocol.Catalog + + + {05c1c78a-9966-4922-9065-a099023e7366} + NgTests + + + + + + + + 1.0.8.3533 + + + 1.0.6 + + + 3.1.0 + + + 4.10.1 + + + 2.75.0 + + + 2.4.1 + + + 2.4.1 + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + + \ No newline at end of file diff --git a/tests/CatalogTests/CollectorHttpClientTests.cs b/tests/CatalogTests/CollectorHttpClientTests.cs new file mode 100644 index 000000000..5af408370 --- /dev/null +++ b/tests/CatalogTests/CollectorHttpClientTests.cs @@ -0,0 +1,108 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Moq; +using Newtonsoft.Json.Linq; +using NgTests.Infrastructure; +using NuGet.Services.Metadata.Catalog; +using Xunit; + +namespace CatalogTests +{ + public class CollectorHttpClientTests + { + private const string TestRawJson = "{\"key\": \"value\"}"; + private const string TestRelativePath = "/index.json"; + private static readonly Uri TestUri = new Uri("http://localhost" + TestRelativePath); + + private readonly MockServerHttpClientHandler _handler; + private readonly Mock _telemetryService; + private readonly CollectorHttpClient _target; + + public CollectorHttpClientTests() + { + _telemetryService = new Mock(); + _handler = new MockServerHttpClientHandler(); + + _target = new CollectorHttpClient(_handler); + } + + private void AddResponse(HttpStatusCode statusCode) + { + _handler.SetAction(TestRelativePath, _ => Task.FromResult(new HttpResponseMessage(statusCode) + { + Content = new StringContent(TestRawJson), + })); + } + + [Fact] + public async Task GetJObjectAsync_WhenStatusIsOK_ReturnsParsedJson() + { + // Arrange + AddResponse(HttpStatusCode.OK); + + // Act + var json = await _target.GetJObjectAsync(TestUri); + + // Assert + Assert.Equal(JObject.Parse(TestRawJson), json); + } + + [Fact] + public async Task GetJObjectAsync_WhenStatusIsNotFound_Throws() + { + AddResponse(HttpStatusCode.NotFound); + + var exception = await Assert.ThrowsAsync(() => _target.GetJObjectAsync(TestUri)); + + Assert.Equal($"GetStringAsync({TestUri})", exception.Message); + Assert.Equal("Response status code does not indicate success: 404 (Not Found).", exception.InnerException.Message); + } + + [Fact] + public async Task GetJObjectAsync_WhenStatusIsInternalServerErrorAndFailureIsPersistent_Throws() + { + AddResponse(HttpStatusCode.InternalServerError); + + var exception = await Assert.ThrowsAsync(() => _target.GetJObjectAsync(TestUri)); + + Assert.Equal($"GetStringAsync({TestUri})", exception.Message); + Assert.Equal("Maximum retry attempts exhausted.", exception.InnerException.Message); + } + + [Fact] + public async Task GetJObjectAsync_WhenStatusIsFirstInternalServerErrorThenOK_ReturnsParsedJson() + { + _handler.SetAction(TestRelativePath, _ => + { + _handler.Actions.Clear(); + + AddResponse(HttpStatusCode.OK); + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.InternalServerError)); + }); + + var json = await _target.GetJObjectAsync(TestUri); + + Assert.Equal(JObject.Parse(TestRawJson), json); + } + + [Fact] + public async Task GetJObjectAsync_WhenStatusIsOKButResponseIsNotJson_Throws() + { + _handler.SetAction(TestRelativePath, _ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("") + })); + + var exception = await Assert.ThrowsAsync(() => _target.GetJObjectAsync(TestUri)); + + Assert.Equal($"GetJObjectAsync({TestUri})", exception.Message); + Assert.Equal("Unexpected character encountered while parsing value: <. Path '', line 0, position 0.", exception.InnerException.Message); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/CommitCollectorFacts.cs b/tests/CatalogTests/CommitCollectorFacts.cs new file mode 100644 index 000000000..a9a27c363 --- /dev/null +++ b/tests/CatalogTests/CommitCollectorFacts.cs @@ -0,0 +1,413 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Moq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests.Infrastructure; +using NuGet.Protocol.Catalog; +using NuGet.Services.Metadata.Catalog; +using Xunit; + +namespace CatalogTests +{ + public class CommitCollectorFacts + { + public class RunAsync : Facts + { + /// + /// This is scenario #1 mentioned in . + /// + [Fact] + public async Task FetchesPageWhenBackCursorIsBeforeEnd() + { + Front.Value = DateTime.Parse("2019-11-04T00:30:00"); + Back.Value = DateTime.Parse("2019-11-04T02:30:00"); + + var output = await Target.Object.RunAsync(Front, Back, Token); + + Assert.Equal(OrderedCommitIds.Skip(2).Take(6).ToArray(), Target.Object.Batches.Select(x => x.Single().CommitId).ToArray()); + HttpRetryStrategy.Verify( + x => x.SendAsync(It.IsAny(), It.IsAny(), It.IsAny()), + Times.Exactly(4)); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/index.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page0.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page1.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page2.json"), It.IsAny())); + } + + /// + /// This is scenario #2 mentioned in . + /// + [Fact] + public async Task FetchesUpToNextPageWhenBackCursorIsAtTheEndOfTheNonLastPage() + { + Front.Value = DateTime.Parse("2019-11-04T01:01:00"); + Back.Value = DateTime.Parse("2019-11-04T03:00:00"); + + var output = await Target.Object.RunAsync(Front, Back, Token); + + Assert.Equal(OrderedCommitIds.Skip(4).Take(5).ToArray(), Target.Object.Batches.Select(x => x.Single().CommitId).ToArray()); + HttpRetryStrategy.Verify( + x => x.SendAsync(It.IsAny(), It.IsAny(), It.IsAny()), + Times.Exactly(3)); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/index.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page1.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page2.json"), It.IsAny())); + } + + /// + /// This is scenario #3 mentioned in . + /// + [Fact] + public async Task SkipsPagesWhenFrontCursorIsAfterEnd() + { + Front.Value = DateTime.Parse("2019-11-04T01:00:00"); + Back.Value = DateTime.Parse("2019-11-04T02:01:00"); + + var output = await Target.Object.RunAsync(Front, Back, Token); + + Assert.Equal(OrderedCommitIds.Skip(3).Take(4).ToArray(), Target.Object.Batches.Select(x => x.Single().CommitId).ToArray()); + HttpRetryStrategy.Verify( + x => x.SendAsync(It.IsAny(), It.IsAny(), It.IsAny()), + Times.Exactly(3)); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/index.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page1.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page2.json"), It.IsAny())); + } + + [Fact] + public async Task FetchesUpToNextPageWhenBackCursorIsAtTheBeginningOfTheNonLastPage() + { + Front.Value = DateTime.Parse("2019-11-04T00:30:00"); + Back.Value = DateTime.Parse("2019-11-04T01:01:00"); + + var output = await Target.Object.RunAsync(Front, Back, Token); + + Assert.Equal(OrderedCommitIds.Skip(2).Take(2).ToArray(), Target.Object.Batches.Select(x => x.Single().CommitId).ToArray()); + HttpRetryStrategy.Verify( + x => x.SendAsync(It.IsAny(), It.IsAny(), It.IsAny()), + Times.Exactly(3)); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/index.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page0.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page1.json"), It.IsAny())); + } + + [Fact] + public async Task FetchesUpToNextPageWhenBackCursorIsInTheMiddleOfTheNonLastPage() + { + Front.Value = DateTime.Parse("2019-11-04T00:30:00"); + Back.Value = DateTime.Parse("2019-11-04T01:30:00"); + + var output = await Target.Object.RunAsync(Front, Back, Token); + + Assert.Equal(OrderedCommitIds.Skip(2).Take(3).ToArray(), Target.Object.Batches.Select(x => x.Single().CommitId).ToArray()); + HttpRetryStrategy.Verify( + x => x.SendAsync(It.IsAny(), It.IsAny(), It.IsAny()), + Times.Exactly(3)); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/index.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page0.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page1.json"), It.IsAny())); + } + + [Fact] + public async Task CanFetchTheEntireCatalog() + { + var output = await Target.Object.RunAsync(Front, Back, Token); + + Assert.Equal(OrderedCommitIds.ToArray(), Target.Object.Batches.Select(x => x.Single().CommitId).ToArray()); + HttpRetryStrategy.Verify( + x => x.SendAsync(It.IsAny(), It.IsAny(), It.IsAny()), + Times.Exactly(5)); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/index.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page0.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page1.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page2.json"), It.IsAny())); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/page3.json"), It.IsAny())); + } + + [Fact] + public async Task ProcessesNoBatchesWhenCursorsAreCaughtUp() + { + Front.Value = DateTime.Parse("2019-11-04T04:00:00"); + + var output = await Target.Object.RunAsync(Front, Back, Token); + + Assert.Empty(Target.Object.Batches); + HttpRetryStrategy.Verify( + x => x.SendAsync(It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + HttpRetryStrategy.Verify(x => x.SendAsync(It.IsAny(), new Uri("https://example/catalog/index.json"), It.IsAny())); + } + } + + public abstract class Facts + { + public Facts() + { + Index = new Uri("https://example/catalog/index.json"); + TelemetryService = new Mock(); + Responses = new Dictionary(); + HttpRetryStrategy = new Mock(); + + Front = MemoryCursor.CreateMin(); + Back = MemoryCursor.CreateMax(); + Token = CancellationToken.None; + HttpRetryStrategy + .Setup(x => x.SendAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((h, u, c) => h.GetAsync(u, c)); + + OrderedCommitIds = new List + { + "51dd5ab3-d1e1-44db-bb94-7142685e4cb8", + "3219e7cd-e79d-400e-bfb8-74e35a07ee77", + "c426811f-9de9-4b4b-b206-73467eff859c", + "edcd04d8-9b9f-44b9-bfb5-0da2b5255487", + "de388ce4-f119-440c-97ad-e6a93e1a516b", + "c59d70fd-deeb-4afd-965a-a902add4e46f", + "7e5797a5-47b8-45af-9603-e7a120ebf7af", + "fe5850b6-e914-495e-87a0-14b9b9030a79", + "e9c20131-1f20-4a19-9900-7ed5a411a9f8", + "35e15b75-f296-4227-a1c6-4d389ef0d072", + "03eeddce-a058-434a-9046-012cb87da057", + "ddaed609-79ba-42e0-8be8-24fe87ba4647", + }; + + Responses[Index.AbsoluteUri] = Serialize(new CatalogIndex + { + Items = new List + { + new CatalogPageItem + { + Url = "https://example/catalog/page0.json", + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T01:00:00Z"), + }, + new CatalogPageItem + { + Url = "https://example/catalog/page1.json", + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T02:00:00Z"), + }, + new CatalogPageItem + { + Url = "https://example/catalog/page2.json", + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T03:00:00Z"), + }, + new CatalogPageItem + { + Url = "https://example/catalog/page3.json", + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T04:00:00Z"), + }, + } + }); + + Responses["https://example/catalog/page0.json"] = Serialize(new CatalogPage + { + Items = new List + { + new CatalogLeafItem + { + Url = "https://example/catalog/item0.json", + PackageId = "NuGet.Versioning", + PackageVersion = "5.1.0", + Type = CatalogLeafType.PackageDetails, + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T00:01:00Z"), + CommitId = OrderedCommitIds[0], + }, + new CatalogLeafItem + { + Url = "https://example/catalog/item1.json", + PackageId = "NuGet.Versioning", + PackageVersion = "5.2.0", + Type = CatalogLeafType.PackageDetails, + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T00:30:00Z"), + CommitId = OrderedCommitIds[1], + }, + new CatalogLeafItem + { + Url = "https://example/catalog/item2.json", + PackageId = "NuGet.Versioning", + PackageVersion = "5.3.0", + Type = CatalogLeafType.PackageDetails, + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T01:00:00Z"), + CommitId = OrderedCommitIds[2], + }, + }, + Context = PageContext, + }); + + Responses["https://example/catalog/page1.json"] = Serialize(new CatalogPage + { + Items = new List + { + new CatalogLeafItem + { + Url = "https://example/catalog/item3.json", + PackageId = "NuGet.Frameworks", + PackageVersion = "5.1.0", + Type = CatalogLeafType.PackageDetails, + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T01:01:00Z"), + CommitId = OrderedCommitIds[3], + }, + new CatalogLeafItem + { + Url = "https://example/catalog/item4.json", + PackageId = "NuGet.Frameworks", + PackageVersion = "5.2.0", + Type = CatalogLeafType.PackageDetails, + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T01:30:00Z"), + CommitId = OrderedCommitIds[4], + }, + new CatalogLeafItem + { + Url = "https://example/catalog/item5.json", + PackageId = "NuGet.Frameworks", + PackageVersion = "5.3.0", + Type = CatalogLeafType.PackageDetails, + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T02:00:00Z"), + CommitId = OrderedCommitIds[5], + }, + }, + Context = PageContext, + }); + + Responses["https://example/catalog/page2.json"] = Serialize(new CatalogPage + { + Items = new List + { + new CatalogLeafItem + { + Url = "https://example/catalog/item6.json", + PackageId = "NuGet.Protocol", + PackageVersion = "5.1.0", + Type = CatalogLeafType.PackageDetails, + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T02:01:00Z"), + CommitId = OrderedCommitIds[6], + }, + new CatalogLeafItem + { + Url = "https://example/catalog/item7.json", + PackageId = "NuGet.Protocol", + PackageVersion = "5.2.0", + Type = CatalogLeafType.PackageDetails, + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T02:30:00Z"), + CommitId = OrderedCommitIds[7], + }, + new CatalogLeafItem + { + Url = "https://example/catalog/item8.json", + PackageId = "NuGet.Protocol", + PackageVersion = "5.3.0", + Type = CatalogLeafType.PackageDetails, + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T03:00:00Z"), + CommitId = OrderedCommitIds[8], + }, + }, + Context = PageContext, + }); + + Responses["https://example/catalog/page3.json"] = Serialize(new CatalogPage + { + Items = new List + { + new CatalogLeafItem + { + Url = "https://example/catalog/item9.json", + PackageId = "NuGet.Commands", + PackageVersion = "5.1.0", + Type = CatalogLeafType.PackageDetails, + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T03:01:00Z"), + CommitId = OrderedCommitIds[9], + }, + new CatalogLeafItem + { + Url = "https://example/catalog/item10.json", + PackageId = "NuGet.Commands", + PackageVersion = "5.2.0", + Type = CatalogLeafType.PackageDetails, + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T03:30:00Z"), + CommitId = OrderedCommitIds[10], + }, + new CatalogLeafItem + { + Url = "https://example/catalog/item11.json", + PackageId = "NuGet.Commands", + PackageVersion = "5.3.0", + Type = CatalogLeafType.PackageDetails, + CommitTimestamp = DateTimeOffset.Parse("2019-11-04T04:00:00Z"), + CommitId = OrderedCommitIds[11], + }, + }, + Context = PageContext, + }); + + Target = new Mock( + Index, + TelemetryService.Object, + (Func)(() => new InMemoryHttpHandler(Responses)), + TimeSpan.FromSeconds(30), + HttpRetryStrategy.Object) + { + CallBase = true, + }; + } + + public Uri Index { get; } + public Mock TelemetryService { get; } + public Dictionary Responses { get; } + public Mock HttpRetryStrategy { get; } + public MemoryCursor Front { get; } + public MemoryCursor Back { get; } + public CancellationToken Token { get; } + public Mock Target { get; } + + public CatalogPageContext PageContext => new CatalogPageContext + { + Vocab = "http://schema.nuget.org/catalog#", + NuGet = "http://schema.nuget.org/schema#", + }; + + public List OrderedCommitIds { get; } + + public string Serialize(T obj) + { + var settings = NuGetJsonSerialization.Settings; + settings.Formatting = Formatting.Indented; + return JsonConvert.SerializeObject(obj, settings); + } + } + + public class TestableCommitCollector : CommitCollector + { + public TestableCommitCollector( + Uri index, + ITelemetryService telemetryService, + Func handlerFunc, + TimeSpan? httpClientTimeout, + IHttpRetryStrategy httpRetryStrategy) + : base(index, telemetryService, handlerFunc, httpClientTimeout, httpRetryStrategy) + { + } + + public List> Batches { get; } = new List>(); + + protected override Task OnProcessBatchAsync( + CollectorHttpClient client, + IEnumerable items, + JToken context, + DateTime commitTimeStamp, + bool isLastBatch, + CancellationToken cancellationToken) + { + Batches.Add(items); + + return Task.FromResult(true); + } + } + } +} diff --git a/tests/CatalogTests/Dnx/DnxCatalogCollectorTests.cs b/tests/CatalogTests/Dnx/DnxCatalogCollectorTests.cs new file mode 100644 index 000000000..94ffcee69 --- /dev/null +++ b/tests/CatalogTests/Dnx/DnxCatalogCollectorTests.cs @@ -0,0 +1,1248 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using CatalogTests.Helpers; +using Microsoft.Extensions.Logging; +using Moq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests; +using NgTests.Data; +using NgTests.Infrastructure; +using NuGet.Protocol.Catalog; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Dnx; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests.Dnx +{ + public class DnxCatalogCollectorTests + { + private static readonly Uri _baseUri = new Uri("https://nuget.test"); + private const string _nuspecData = "nuspec data"; + private const int _maxDegreeOfParallelism = 20; + private static readonly HttpContent _noContent = new ByteArrayContent(new byte[0]); + private const IAzureStorage _nullPreferredPackageSourceStorage = null; + private static readonly Uri _contentBaseAddress = new Uri("http://tempuri.org/packages/"); + private readonly JObject _contextKeyword = new JObject( + new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetSchemaUri), + new JProperty(CatalogConstants.NuGet, CatalogConstants.NuGetSchemaUri), + new JProperty(CatalogConstants.Items, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Item), + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.Parent, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), + new JProperty(CatalogConstants.CommitTimeStamp, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.NuGetLastCreated, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.NuGetLastEdited, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.NuGetLastDeleted, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime)))); + private readonly JsonSerializerSettings _jsonSettings = new JsonSerializerSettings() + { + DateParseHandling = DateParseHandling.None, + NullValueHandling = NullValueHandling.Ignore + }; + + private MemoryStorage _catalogToDnxStorage; + private TestStorageFactory _catalogToDnxStorageFactory; + private MockServerHttpClientHandler _mockServer; + private ILogger _logger; + private DnxCatalogCollector _target; + private Random _random; + private Uri _cursorJsonUri; + private Mock _mockCatalogClient; + + public DnxCatalogCollectorTests() + { + _catalogToDnxStorage = new MemoryStorage(_baseUri); + _catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name)); + _mockServer = new MockServerHttpClientHandler(); + _mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK))); + _mockCatalogClient = new Mock(); + _mockCatalogClient + .Setup(cc => cc.GetPackageDetailsLeafAsync(It.IsAny())) + .ReturnsAsync(new PackageDetailsCatalogLeaf()); + + var loggerFactory = new LoggerFactory(); + _logger = loggerFactory.CreateLogger(); + + _target = new DnxCatalogCollector( + new Uri("http://tempuri.org/index.json"), + _catalogToDnxStorageFactory, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), + _logger, + _maxDegreeOfParallelism, + _ => _mockCatalogClient.Object, + () => _mockServer); + + _cursorJsonUri = _catalogToDnxStorage.ResolveUri("cursor.json"); + _random = new Random(); + } + + [Fact] + public void Constructor_WhenIndexIsNull_Throws() + { + Uri index = null; + + using (var clientHandler = new HttpClientHandler()) + { + var exception = Assert.Throws( + () => new DnxCatalogCollector( + index, + new TestStorageFactory(), + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), + Mock.Of(), + maxDegreeOfParallelism: 1, + catalogClientFactory: _ => _mockCatalogClient.Object, + handlerFunc: () => clientHandler, + httpClientTimeout: TimeSpan.Zero)); + + Assert.Equal("index", exception.ParamName); + } + } + + [Fact] + public void Constructor_WhenStorageFactoryIsNull_Throws() + { + StorageFactory storageFactory = null; + + using (var clientHandler = new HttpClientHandler()) + { + var exception = Assert.Throws( + () => new DnxCatalogCollector( + new Uri("https://nuget.test"), + storageFactory, + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: Mock.Of(), + logger: Mock.Of(), + maxDegreeOfParallelism: 1, + catalogClientFactory: _ => _mockCatalogClient.Object, + handlerFunc: () => clientHandler, + httpClientTimeout: TimeSpan.Zero)); + + Assert.Equal("storageFactory", exception.ParamName); + } + } + + [Fact] + public void Constructor_WhenTelemetryServiceIsNull_Throws() + { + using (var clientHandler = new HttpClientHandler()) + { + var exception = Assert.Throws( + () => new DnxCatalogCollector( + new Uri("https://nuget.test"), + new TestStorageFactory(), + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: null, + logger: Mock.Of(), + maxDegreeOfParallelism: 1, + catalogClientFactory: _ => _mockCatalogClient.Object, + handlerFunc: () => clientHandler, + httpClientTimeout: TimeSpan.Zero)); + + Assert.Equal("telemetryService", exception.ParamName); + } + } + + [Fact] + public void Constructor_WhenLoggerIsNull_Throws() + { + ILogger logger = null; + + using (var clientHandler = new HttpClientHandler()) + { + var exception = Assert.Throws( + () => new DnxCatalogCollector( + new Uri("https://nuget.test"), + new TestStorageFactory(), + _nullPreferredPackageSourceStorage, + null, + Mock.Of(), + logger, + maxDegreeOfParallelism: 1, + catalogClientFactory: _ => _mockCatalogClient.Object, + handlerFunc: () => clientHandler, + httpClientTimeout: TimeSpan.Zero)); + + Assert.Equal("logger", exception.ParamName); + } + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void Constructor_WhenMaxDegreeOfParallelismIsLessThanOne_Throws(int maxDegreeOfParallelism) + { + using (var clientHandler = new HttpClientHandler()) + { + var exception = Assert.Throws( + () => new DnxCatalogCollector( + new Uri("https://nuget.test"), + new TestStorageFactory(), + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: Mock.Of(), + logger: Mock.Of(), + maxDegreeOfParallelism: maxDegreeOfParallelism, + catalogClientFactory: _ => _mockCatalogClient.Object, + handlerFunc: () => clientHandler, + httpClientTimeout: TimeSpan.Zero)); + + Assert.Equal("maxDegreeOfParallelism", exception.ParamName); + Assert.StartsWith($"The argument must be within the range from 1 (inclusive) to {int.MaxValue} (inclusive).", exception.Message); + } + } + + [Fact] + public void Constructor_WhenCatalogClientFactoryIsNull_Throws() + { + using (var clientHandler = new HttpClientHandler()) + { + var exception = Assert.Throws( + () => new DnxCatalogCollector( + new Uri("https://nuget.test"), + new TestStorageFactory(), + _nullPreferredPackageSourceStorage, + null, + Mock.Of(), + logger: Mock.Of(), + maxDegreeOfParallelism: 1, + catalogClientFactory: null, + handlerFunc: () => clientHandler, + httpClientTimeout: TimeSpan.Zero)); + + Assert.Equal("catalogClientFactory", exception.ParamName); + } + } + + [Fact] + public void Constructor_WhenHandlerFuncIsNull_InstantiatesClass() + { + new DnxCatalogCollector( + new Uri("https://nuget.test"), + new TestStorageFactory(), + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: Mock.Of(), + logger: Mock.Of(), + maxDegreeOfParallelism: 1, + catalogClientFactory: _ => _mockCatalogClient.Object, + handlerFunc: null, + httpClientTimeout: TimeSpan.Zero); + } + + [Fact] + public void Constructor_WhenHttpClientTimeoutIsNull_InstantiatesClass() + { + using (var clientHandler = new HttpClientHandler()) + { + new DnxCatalogCollector( + new Uri("https://nuget.test"), + new TestStorageFactory(), + _nullPreferredPackageSourceStorage, + contentBaseAddress: null, + telemetryService: Mock.Of(), + logger: Mock.Of(), + maxDegreeOfParallelism: 1, + catalogClientFactory: _ => _mockCatalogClient.Object, + handlerFunc: () => clientHandler, + httpClientTimeout: null); + } + } + + [Fact] + public async Task RunAsync_WhenPackageDoesNotHaveNuspec_SkipsPackage() + { + var zipWithNoNuspec = CreateZipStreamWithEntry("readme.txt", "content"); + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); + var catalogStorage = Catalogs.CreateTestCatalogWithThreePackages(); + + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/listedpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(zipWithNoNuspec) })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.RunAsync(front, back, CancellationToken.None); + + Assert.Single(_catalogToDnxStorage.Content); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(indexJsonUri)); + } + + [Fact] + public async Task RunAsync_WhenPackageHasNuspecWithWrongName_ProcessesPackage() + { + var zipWithWrongNameNuspec = CreateZipStreamWithEntry("Newtonsoft.Json.nuspec", _nuspecData); + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/1.0.0/unlistedpackage.1.0.0.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/1.0.0/unlistedpackage.nuspec"); + var catalogStorage = Catalogs.CreateTestCatalogWithThreePackages(); + + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/unlistedpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(zipWithWrongNameNuspec) })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.RunAsync(front, back, CancellationToken.None); + + Assert.Equal(4, _catalogToDnxStorage.Content.Count); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nupkgUri, out var nupkg)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nuspecUri, out var nuspec)); + + Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson)); + Assert.Equal(zipWithWrongNameNuspec.ToArray(), nupkg); + Assert.Equal(_nuspecData, Encoding.UTF8.GetString(nuspec)); + } + + [Fact] + public async Task RunAsync_WhenSourceNupkgIsNotFound_SkipsPackage() + { + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/1.0.0/unlistedpackage.1.0.0.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/1.0.0/unlistedpackage.nuspec"); + var catalogStorage = Catalogs.CreateTestCatalogWithThreePackages(); + + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/listedpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) { Content = _noContent })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.RunAsync(front, back, CancellationToken.None); + + Assert.Single(_catalogToDnxStorage.Content); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(indexJsonUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); + } + + [Fact] + public async Task RunAsync_WithValidPackage_CreatesFlatContainer() + { + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec"); + var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); + var expectedNupkg = File.ReadAllBytes("Packages\\ListedPackage.1.0.0.zip"); + + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/listedpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(expectedNupkg) })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.RunAsync(front, back, CancellationToken.None); + + Assert.Equal(4, _catalogToDnxStorage.Content.Count); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nupkgUri, out var nupkg)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nuspecUri, out var nuspec)); + + Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson)); + Assert.Equal(expectedNupkg, nupkg); + Assert.Equal( + "\r\n\r\n \r\n ListedPackage\r\n 1.0.0\r\n NuGet\r\n false\r\n Package description.\r\n \r\n", + Encoding.UTF8.GetString(nuspec)); + } + + [Fact] + public async Task RunAsync_WithValidPackage_RespectsDeletion() + { + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/otherpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/otherpackage/1.0.0/otherpackage.1.0.0.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/otherpackage/1.0.0/otherpackage.nuspec"); + var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); + + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/otherpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\OtherPackage.1.0.0.zip")) })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.RunAsync(front, back, CancellationToken.None); + + Assert.Single(_catalogToDnxStorage.Content); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(indexJsonUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); + } + + [Fact] + public async Task RunAsync_WithPackageCreatedThenDeleted_LeavesNoArtifacts() + { + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/otherpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/otherpackage/1.0.0/otherpackage.1.0.0.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/otherpackage/1.0.0/otherpackage.nuspec"); + var catalogStorage = Catalogs.CreateTestCatalogWithPackageCreatedThenDeleted(); + + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/otherpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\OtherPackage.1.0.0.zip")) })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.RunAsync(front, back, CancellationToken.None); + + Assert.Single(_catalogToDnxStorage.Content); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(indexJsonUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); + } + + [Fact] + public async Task RunAsync_WithNonIAzureStorage_WhenPackageIsAlreadySynchronizedAndHasRequiredProperties_SkipsPackage() + { + _catalogToDnxStorage = new SynchronizedMemoryStorage(new[] + { + new Uri("http://tempuri.org/packages/listedpackage.1.0.1.nupkg"), + }); + _catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name)); + + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.nuspec"); + var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.1.zip"); + + await _catalogToDnxStorage.SaveAsync( + new Uri("http://tempuri.org/listedpackage/index.json"), + new StringStorageContent(GetExpectedIndexJsonContent("1.0.1")), + CancellationToken.None); + + _target = new DnxCatalogCollector( + new Uri("http://tempuri.org/index.json"), + _catalogToDnxStorageFactory, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), + _logger, + _maxDegreeOfParallelism, + _ => _mockCatalogClient.Object, + () => _mockServer); + + var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/listedpackage.1.0.1.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(nupkgStream) })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.RunAsync(front, back, CancellationToken.None); + + Assert.Equal(2, _catalogToDnxStorage.Content.Count); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); + + Assert.Equal(GetExpectedIndexJsonContent("1.0.1"), Encoding.UTF8.GetString(indexJson)); + } + + [Fact] + public async Task RunAsync_WithIAzureStorage_WhenPackageIsAlreadySynchronizedAndHasRequiredProperties_SkipsPackage() + { + _catalogToDnxStorage = new AzureSynchronizedMemoryStorageStub(new[] + { + new Uri("http://tempuri.org/packages/listedpackage.1.0.0.nupkg") + }, areRequiredPropertiesPresentAsync: true); + _catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name)); + + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec"); + var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.0.zip"); + + await _catalogToDnxStorage.SaveAsync( + new Uri("http://tempuri.org/listedpackage/index.json"), + new StringStorageContent(GetExpectedIndexJsonContent("1.0.0")), + CancellationToken.None); + + _target = new DnxCatalogCollector( + new Uri("http://tempuri.org/index.json"), + _catalogToDnxStorageFactory, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), + _logger, + _maxDegreeOfParallelism, + _ => _mockCatalogClient.Object, + () => _mockServer); + + var catalogStorage = Catalogs.CreateTestCatalogWithOnePackage(); + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/listedpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(nupkgStream) })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.RunAsync(front, back, CancellationToken.None); + + Assert.Equal(2, _catalogToDnxStorage.Content.Count); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); + + Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson)); + } + + [Fact] + public async Task RunAsync_WithFakeIAzureStorage_WhenPackageIsAlreadySynchronizedButDoesNotHaveRequiredProperties_ProcessesPackage() + { + _catalogToDnxStorage = new AzureSynchronizedMemoryStorageStub(new[] + { + new Uri("http://tempuri.org/packages/listedpackage.1.0.0.nupkg") + }, areRequiredPropertiesPresentAsync: false); + _catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name)); + + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec"); + var expectedNupkg = File.ReadAllBytes("Packages\\ListedPackage.1.0.0.zip"); + + await _catalogToDnxStorage.SaveAsync( + new Uri("http://tempuri.org/listedpackage/index.json"), + new StringStorageContent(GetExpectedIndexJsonContent("1.0.0")), + CancellationToken.None); + + _target = new DnxCatalogCollector( + new Uri("http://tempuri.org/index.json"), + _catalogToDnxStorageFactory, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), + _logger, + _maxDegreeOfParallelism, + _ => _mockCatalogClient.Object, + () => _mockServer); + + var catalogStorage = Catalogs.CreateTestCatalogWithOnePackage(); + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/listedpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(expectedNupkg) })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.RunAsync(front, back, CancellationToken.None); + + Assert.Equal(4, _catalogToDnxStorage.Content.Count); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); + + Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson)); + } + + [Fact] + public async Task RunAsync_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPackage() + { + _catalogToDnxStorage = new SynchronizedMemoryStorage(new[] + { + new Uri("http://tempuri.org/packages/listedpackage.1.0.1.nupkg"), + }); + _catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name)); + _mockServer = new MockServerHttpClientHandler(); + _mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK))); + + var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.nuspec"); + + _target = new DnxCatalogCollector( + new Uri("http://tempuri.org/index.json"), + _catalogToDnxStorageFactory, + _nullPreferredPackageSourceStorage, + _contentBaseAddress, + Mock.Of(), + new Mock().Object, + _maxDegreeOfParallelism, + _ => _mockCatalogClient.Object, + () => _mockServer); + + var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); + + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/listedpackage.1.0.1.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.1.zip")) })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.RunAsync(front, back, CancellationToken.None); + + Assert.Equal(2, _catalogToDnxStorage.Content.Count); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); + Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); + + Assert.Equal(GetExpectedIndexJsonContent("1.0.1"), Encoding.UTF8.GetString(indexJson)); + } + + [Theory] + [InlineData(HttpStatusCode.BadRequest)] + [InlineData(HttpStatusCode.Unauthorized)] + [InlineData(HttpStatusCode.Forbidden)] + [InlineData(HttpStatusCode.NoContent)] + [InlineData(HttpStatusCode.InternalServerError)] + [InlineData(HttpStatusCode.ServiceUnavailable)] + public async Task RunAsync_WhenDownloadingPackage_RejectsUnexpectedHttpStatusCode(HttpStatusCode statusCode) + { + var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); + + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.Return404OnUnknownAction = true; + + _mockServer.SetAction( + "/packages/listedpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(statusCode) { Content = _noContent })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + var exception = await Assert.ThrowsAsync( + () => _target.RunAsync(front, back, CancellationToken.None)); + Assert.IsType(exception.InnerException); + Assert.Equal( + $"Expected status code OK for package download, actual: {statusCode}", + exception.InnerException.Message); + Assert.Empty(_catalogToDnxStorage.Content); + } + + [Fact] + public async Task RunAsync_WhenDownloadingPackage_OnlyDownloadsNupkgOncePerCatalogLeaf() + { + // Arrange + var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/listedpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.0.zip")) })); + _mockServer.SetAction( + "/packages/listedpackage.1.0.1.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.1.zip")) })); + _mockServer.SetAction( + "/packages/unlistedpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\UnlistedPackage.1.0.0.zip")) })); + _mockServer.SetAction( + "/packages/otherpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\OtherPackage.1.0.0.zip")) })); + + ReadWriteCursor front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + // Act + await _target.RunAsync(front, back, CancellationToken.None); + + // Assert + Assert.Equal(9, _catalogToDnxStorage.Content.Count); + + Assert.Equal(5, _mockServer.Requests.Count); + Assert.EndsWith("/index.json", _mockServer.Requests[0].RequestUri.AbsoluteUri); + Assert.EndsWith("/page0.json", _mockServer.Requests[1].RequestUri.AbsoluteUri); + + // The packages were processed in random order. + var remainingRequests = _mockServer.Requests + .Skip(2) + .Take(3) + .Select(request => request.RequestUri.AbsoluteUri) + .OrderBy(uri => uri) + .ToArray(); + + Assert.Contains("/listedpackage.1.0.0.nupkg", remainingRequests[0]); + Assert.Contains("/listedpackage.1.0.1.nupkg", remainingRequests[1]); + Assert.Contains("/unlistedpackage.1.0.0.nupkg", remainingRequests[2]); + } + + [Theory] + [InlineData("/packages/unlistedpackage.1.0.0.nupkg", null)] + [InlineData("/packages/listedpackage.1.0.1.nupkg", "2015-10-12T10:08:54.1506742Z")] + [InlineData("/packages/anotherpackage.1.0.0.nupkg", "2015-10-12T10:08:54.1506742Z")] + public async Task RunAsync_WhenExceptionOccurs_DoesNotSkipPackage(string catalogUri, string expectedCursorBeforeRetry) + { + // Arrange + var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit(); + await _mockServer.AddStorageAsync(catalogStorage); + + _mockServer.SetAction( + "/packages/unlistedpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\UnlistedPackage.1.0.0.zip")) })); + _mockServer.SetAction( + "/packages/listedpackage.1.0.1.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.1.zip")) })); + _mockServer.SetAction( + "/packages/anotherpackage.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.0.zip")) })); + + // Make the first request for a catalog leaf node fail. This will cause the registration collector + // to fail the first time but pass the second time. + FailFirstRequest(catalogUri); + + expectedCursorBeforeRetry = expectedCursorBeforeRetry ?? MemoryCursor.MinValue.ToString("O"); + + var front = new DurableCursor( + _cursorJsonUri, + _catalogToDnxStorage, + MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + // Act + var exception = await Assert.ThrowsAsync( + () => _target.RunAsync(front, back, CancellationToken.None)); + Assert.IsType(exception.InnerException); + var cursorBeforeRetry = front.Value; + await _target.RunAsync(front, back, CancellationToken.None); + var cursorAfterRetry = front.Value; + + // Assert + var unlistedPackage100 = _catalogToDnxStorage + .Content + .FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0/unlistedpackage.1.0.0.nupkg")); + Assert.NotNull(unlistedPackage100.Key); + + var listedPackage101 = _catalogToDnxStorage + .Content + .FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg")); + Assert.NotNull(listedPackage101.Key); + + var anotherPackage100 = _catalogToDnxStorage + .Content + .FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/anotherpackage/1.0.0/anotherpackage.1.0.0.nupkg")); + Assert.NotNull(anotherPackage100.Key); + + Assert.Equal(MemoryCursor.MinValue, cursorBeforeRetry); + Assert.Equal(DateTime.Parse("2015-10-12T10:08:55.3335317Z").ToUniversalTime(), cursorAfterRetry); + } + + [Fact] + public async Task RunAsync_WithIdenticalCommitItems_ProcessesPackage() + { + using (var package = TestPackage.Create(_random)) + { + var catalogIndexUri = new Uri(_baseUri, "index.json"); + var catalogPageUri = new Uri(_baseUri, "page0.json"); + + var commitId = Guid.NewGuid().ToString(); + var commitTimeStamp = DateTimeOffset.UtcNow; + var independentPackageDetails0 = new CatalogIndependentPackageDetails( + package.Id, + package.Version.ToNormalizedString(), + _baseUri.AbsoluteUri, + commitId, + commitTimeStamp); + var independentPackageDetails1 = new CatalogIndependentPackageDetails( + package.Id, + package.Version.ToNormalizedString(), + _baseUri.AbsoluteUri, + commitId, + commitTimeStamp); + var packageDetails = new[] + { + CatalogPackageDetails.Create(independentPackageDetails0), + CatalogPackageDetails.Create(independentPackageDetails1) + }; + + var independentPage = new CatalogIndependentPage( + catalogPageUri.AbsoluteUri, + CatalogConstants.CatalogPage, + commitId, + commitTimeStamp.ToString(CatalogConstants.CommitTimeStampFormat), + packageDetails.Length, + catalogIndexUri.AbsoluteUri, + packageDetails, + _contextKeyword); + + var index = Helpers.CatalogIndex.Create(independentPage, _contextKeyword); + var catalogStorage = new MemoryStorage(_baseUri); + + catalogStorage.Content.TryAdd(catalogIndexUri, CreateStringStorageContent(index)); + catalogStorage.Content.TryAdd(catalogPageUri, CreateStringStorageContent(independentPage)); + catalogStorage.Content.TryAdd( + new Uri(independentPackageDetails0.IdKeyword), + CreateStringStorageContent(independentPackageDetails0)); + catalogStorage.Content.TryAdd( + new Uri(independentPackageDetails1.IdKeyword), + CreateStringStorageContent(independentPackageDetails1)); + + byte[] expectedNupkgBytes = ReadPackageBytes(package); + + await _mockServer.AddStorageAsync(catalogStorage); + + var packageId = package.Id.ToLowerInvariant(); + var packageVersion = package.Version.ToNormalizedString().ToLowerInvariant(); + var nupkgPathAndQuery = $"/packages/{packageId}.{packageVersion}.nupkg"; + + _mockServer.SetAction( + nupkgPathAndQuery, + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(expectedNupkgBytes) + })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.RunAsync(front, back, CancellationToken.None); + + Assert.Equal(4, _catalogToDnxStorage.Content.Count); + Assert.Equal(3, _mockServer.Requests.Count); + + Assert.EndsWith("/index.json", _mockServer.Requests[0].RequestUri.AbsoluteUri); + Assert.EndsWith("/page0.json", _mockServer.Requests[1].RequestUri.AbsoluteUri); + Assert.Contains(nupkgPathAndQuery, _mockServer.Requests[2].RequestUri.AbsoluteUri); + + var indexJsonUri = _catalogToDnxStorage.ResolveUri($"{packageId}/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri($"{packageId}/{packageVersion}/{packageId}.{packageVersion}.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri($"{packageId}/{packageVersion}/{packageId}.nuspec"); + + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(_cursorJsonUri, out var cursorBytes)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexBytes)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nupkgUri, out var actualNupkgBytes)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nuspecUri, out var nuspecBytes)); + + var actualCursorJson = Encoding.UTF8.GetString(cursorBytes); + var actualIndexJson = Encoding.UTF8.GetString(indexBytes); + var actualNuspec = Encoding.UTF8.GetString(nuspecBytes); + + Assert.Equal(GetExpectedCursorJsonContent(front.Value.ToString("yyyy-MM-ddTHH:mm:ss.fffffff")), actualCursorJson); + Assert.Equal(GetExpectedIndexJsonContent(packageVersion), actualIndexJson); + Assert.Equal(expectedNupkgBytes, actualNupkgBytes); + Assert.Equal(package.Nuspec, actualNuspec); + } + } + + [Fact] + public async Task RunAsync_WhenMultipleCommitItemsWithSamePackageIdentityExistAcrossMultipleCommits_OnlyLastCommitIsProcessed() + { + using (var package = TestPackage.Create(_random)) + { + var catalogIndexUri = new Uri(_baseUri, "index.json"); + var catalogPageUri = new Uri(_baseUri, "page0.json"); + + var commitTimeStamp1 = DateTimeOffset.UtcNow; + var commitTimeStamp0 = commitTimeStamp1.AddMinutes(-1); + var independentPackageDetails0 = new CatalogIndependentPackageDetails( + package.Id, + package.Version.ToNormalizedString(), + _baseUri.AbsoluteUri, + Guid.NewGuid().ToString(), + commitTimeStamp0); + var independentPackageDetails1 = new CatalogIndependentPackageDetails( + package.Id, + package.Version.ToNormalizedString(), + _baseUri.AbsoluteUri, + Guid.NewGuid().ToString(), + commitTimeStamp1); + var packageDetails = new[] + { + CatalogPackageDetails.Create(independentPackageDetails0), + CatalogPackageDetails.Create(independentPackageDetails1) + }; + + var independentPage = new CatalogIndependentPage( + catalogPageUri.AbsoluteUri, + CatalogConstants.CatalogPage, + independentPackageDetails1.CommitId, + independentPackageDetails1.CommitTimeStamp, + packageDetails.Length, + catalogIndexUri.AbsoluteUri, + packageDetails, + _contextKeyword); + + var index = Helpers.CatalogIndex.Create(independentPage, _contextKeyword); + var catalogStorage = new MemoryStorage(_baseUri); + + catalogStorage.Content.TryAdd(catalogIndexUri, CreateStringStorageContent(index)); + catalogStorage.Content.TryAdd(catalogPageUri, CreateStringStorageContent(independentPage)); + catalogStorage.Content.TryAdd( + new Uri(independentPackageDetails0.IdKeyword), + CreateStringStorageContent(independentPackageDetails0)); + catalogStorage.Content.TryAdd( + new Uri(independentPackageDetails1.IdKeyword), + CreateStringStorageContent(independentPackageDetails1)); + + byte[] expectedNupkgBytes = ReadPackageBytes(package); + + await _mockServer.AddStorageAsync(catalogStorage); + + var packageId = package.Id.ToLowerInvariant(); + var packageVersion = package.Version.ToNormalizedString().ToLowerInvariant(); + var nupkgPathAndQuery = $"/packages/{packageId}.{packageVersion}.nupkg"; + + _mockServer.SetAction( + nupkgPathAndQuery, + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(expectedNupkgBytes) + })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + await _target.RunAsync(front, back, CancellationToken.None); + + Assert.Equal(4, _catalogToDnxStorage.Content.Count); + Assert.Equal(3, _mockServer.Requests.Count); + + Assert.EndsWith("/index.json", _mockServer.Requests[0].RequestUri.AbsoluteUri); + Assert.EndsWith("/page0.json", _mockServer.Requests[1].RequestUri.AbsoluteUri); + Assert.Contains(nupkgPathAndQuery, _mockServer.Requests[2].RequestUri.AbsoluteUri); + + var indexJsonUri = _catalogToDnxStorage.ResolveUri($"{packageId}/index.json"); + var nupkgUri = _catalogToDnxStorage.ResolveUri($"{packageId}/{packageVersion}/{packageId}.{packageVersion}.nupkg"); + var nuspecUri = _catalogToDnxStorage.ResolveUri($"{packageId}/{packageVersion}/{packageId}.nuspec"); + + Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); + Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(_cursorJsonUri, out var cursorBytes)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexBytes)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nupkgUri, out var actualNupkgBytes)); + Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nuspecUri, out var nuspecBytes)); + + var actualCursorJson = Encoding.UTF8.GetString(cursorBytes); + var actualIndexJson = Encoding.UTF8.GetString(indexBytes); + var actualNuspec = Encoding.UTF8.GetString(nuspecBytes); + + Assert.Equal(GetExpectedCursorJsonContent(front.Value.ToString("yyyy-MM-ddTHH:mm:ss.fffffff")), actualCursorJson); + Assert.Equal(GetExpectedIndexJsonContent(packageVersion), actualIndexJson); + Assert.Equal(expectedNupkgBytes, actualNupkgBytes); + Assert.Equal(package.Nuspec, actualNuspec); + } + } + + [Fact] + public async Task RunAsync_WhenMultipleCommitItemsHaveSameCommitTimeStampButDifferentCommitId_Throws() + { + using (var package0 = TestPackage.Create(_random)) + using (var package1 = TestPackage.Create(_random)) + { + var catalogIndexUri = new Uri(_baseUri, "index.json"); + var catalogPageUri = new Uri(_baseUri, "page0.json"); + + var commitId0 = Guid.NewGuid().ToString(); + var commitId1 = Guid.NewGuid().ToString(); + var commitTimeStamp = DateTime.UtcNow; + var independentPackageDetails0 = new CatalogIndependentPackageDetails( + package0.Id, + package0.Version.ToNormalizedString(), + _baseUri.AbsoluteUri, + commitId0, + commitTimeStamp); + var independentPackageDetails1 = new CatalogIndependentPackageDetails( + package1.Id, + package1.Version.ToNormalizedString(), + _baseUri.AbsoluteUri, + commitId1, + commitTimeStamp); + var packageDetails = new[] + { + CatalogPackageDetails.Create(independentPackageDetails0), + CatalogPackageDetails.Create(independentPackageDetails1) + }; + + var independentPage = new CatalogIndependentPage( + catalogPageUri.AbsoluteUri, + CatalogConstants.CatalogPage, + independentPackageDetails1.CommitId, + commitTimeStamp.ToString(CatalogConstants.CommitTimeStampFormat), + packageDetails.Length, + catalogIndexUri.AbsoluteUri, + packageDetails, + _contextKeyword); + + var index = Helpers.CatalogIndex.Create(independentPage, _contextKeyword); + var catalogStorage = new MemoryStorage(_baseUri); + + catalogStorage.Content.TryAdd(catalogIndexUri, CreateStringStorageContent(index)); + catalogStorage.Content.TryAdd(catalogPageUri, CreateStringStorageContent(independentPage)); + catalogStorage.Content.TryAdd( + new Uri(independentPackageDetails0.IdKeyword), + CreateStringStorageContent(independentPackageDetails0)); + catalogStorage.Content.TryAdd( + new Uri(independentPackageDetails1.IdKeyword), + CreateStringStorageContent(independentPackageDetails1)); + + byte[] expectedNupkgBytes = ReadPackageBytes(package0); + + await _mockServer.AddStorageAsync(catalogStorage); + + var packageId = package0.Id.ToLowerInvariant(); + var packageVersion = package0.Version.ToNormalizedString().ToLowerInvariant(); + var nupkgPathAndQuery = $"/packages/{packageId}.{packageVersion}.nupkg"; + + _mockServer.SetAction( + nupkgPathAndQuery, + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(expectedNupkgBytes) + })); + + var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); + ReadCursor back = MemoryCursor.CreateMax(); + + var exception = await Assert.ThrowsAsync( + () => _target.RunAsync(front, back, CancellationToken.None)); + + var expectedMessage = "Multiple commits exist with the same commit timestamp but different commit ID's: " + + $"{{ CommitId = {commitId0}, CommitTimeStamp = {commitTimeStamp.ToString("yyyy-MM-ddTHH:mm:ss.fffffff")} }}, " + + $"{{ CommitId = {commitId1}, CommitTimeStamp = {commitTimeStamp.ToString("yyyy-MM-ddTHH:mm:ss.fffffff")} }}"; + + Assert.StartsWith(expectedMessage, exception.Message); + } + } + + private static string GetExpectedCursorJsonContent(string cursor) + { + return $"{{\r\n \"value\": \"{cursor}\"\r\n}}"; + } + + private static string GetExpectedIndexJsonContent(string version) + { + return $"{{\r\n \"versions\": [\r\n \"{version}\"\r\n ]\r\n}}"; + } + + private void FailFirstRequest(string relativeUri) + { + var originalAction = _mockServer.Actions[relativeUri]; + var hasFailed = false; + Func> failFirst = request => + { + if (!hasFailed) + { + hasFailed = true; + throw new HttpRequestException("Simulated HTTP failure."); + } + + return originalAction(request); + }; + _mockServer.SetAction(relativeUri, failFirst); + } + + private StringStorageContent CreateStringStorageContent(T value) + { + return new StringStorageContent(JsonConvert.SerializeObject(value, _jsonSettings)); + } + + private static MemoryStream CreateZipStreamWithEntry(string name, string content) + { + var zipWithNoNuspec = new MemoryStream(); + + using (var zipArchive = new ZipArchive(zipWithNoNuspec, ZipArchiveMode.Create, leaveOpen: true)) + { + var entry = zipArchive.CreateEntry(name); + + using (var entryStream = entry.Open()) + using (var entryWriter = new StreamWriter(entryStream)) + { + entryWriter.Write(content); + } + } + + zipWithNoNuspec.Position = 0; + + return zipWithNoNuspec; + } + + private static byte[] ReadPackageBytes(TestPackage package) + { + using (var reader = new BinaryReader(package.Stream)) + { + return reader.ReadBytes((int)package.Stream.Length); + } + } + + private class SynchronizedMemoryStorage : MemoryStorage + { + protected HashSet SynchronizedUris { get; private set; } + + public SynchronizedMemoryStorage(IEnumerable synchronizedUris) + { + SynchronizedUris = new HashSet(synchronizedUris); + } + + protected SynchronizedMemoryStorage( + Uri baseAddress, + ConcurrentDictionary content, + ConcurrentDictionary contentBytes, + HashSet synchronizedUris) + : base(baseAddress, content, contentBytes) + { + SynchronizedUris = synchronizedUris; + } + + public override Task AreSynchronized(Uri firstResourceUri, Uri secondResourceUri) + { + return Task.FromResult(SynchronizedUris.Contains(firstResourceUri)); + } + + public override Storage WithName(string name) + { + return new SynchronizedMemoryStorage( + new Uri(BaseAddress + name), + Content, + ContentBytes, + SynchronizedUris); + } + } + + private class AzureSynchronizedMemoryStorageStub : SynchronizedMemoryStorage, IAzureStorage + { + private readonly bool _areRequiredPropertiesPresentAsync; + + internal AzureSynchronizedMemoryStorageStub( + IEnumerable synchronizedUris, + bool areRequiredPropertiesPresentAsync) + : base(synchronizedUris) + { + _areRequiredPropertiesPresentAsync = areRequiredPropertiesPresentAsync; + } + + protected AzureSynchronizedMemoryStorageStub( + Uri baseAddress, + ConcurrentDictionary content, + ConcurrentDictionary contentBytes, + HashSet synchronizedUris, + bool areRequiredPropertiesPresentAsync) + : base(baseAddress, content, contentBytes, synchronizedUris) + { + _areRequiredPropertiesPresentAsync = areRequiredPropertiesPresentAsync; + } + + public override Storage WithName(string name) + { + return new AzureSynchronizedMemoryStorageStub( + new Uri(BaseAddress + name), + Content, + ContentBytes, + SynchronizedUris, + _areRequiredPropertiesPresentAsync); + } + + public Task GetCloudBlockBlobReferenceAsync(Uri blobUri) + { + throw new NotImplementedException(); + } + + public Task GetCloudBlockBlobReferenceAsync(string name) + { + throw new NotImplementedException(); + } + + public Task HasPropertiesAsync(Uri blobUri, string contentType, string cacheControl) + { + return Task.FromResult(_areRequiredPropertiesPresentAsync); + } + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Dnx/DnxMakerTests.cs b/tests/CatalogTests/Dnx/DnxMakerTests.cs new file mode 100644 index 000000000..e1f5f2275 --- /dev/null +++ b/tests/CatalogTests/Dnx/DnxMakerTests.cs @@ -0,0 +1,880 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using Newtonsoft.Json.Linq; +using NgTests.Infrastructure; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Dnx; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Versioning; +using Xunit; + +namespace CatalogTests.Dnx +{ + public class DnxMakerTests + { + private const string _expectedCacheControl = "max-age=120"; + private const string _expectedNuspecContentType = "text/xml"; + private const string _expectedPackageContentType = "application/octet-stream"; + private const string _expectedPackageVersionIndexJsonCacheControl = "no-store"; + private const string _expectedPackageVersionIndexJsonContentType = "application/json"; + private const string _packageId = "testid"; + private const string _nupkgData = "nupkg data"; + private const string _nuspecData = "nuspec data"; + + [Fact] + public void Constructor_WhenStorageFactoryIsNull_Throws() + { + var exception = Assert.Throws( + () => new DnxMaker( + storageFactory: null, + telemetryService: Mock.Of(), + logger: Mock.Of())); + + Assert.Equal("storageFactory", exception.ParamName); + } + + [Fact] + public void Constructor_WhenTelemetryServiceIsNull_Throws() + { + var exception = Assert.Throws( + () => new DnxMaker( + storageFactory: Mock.Of(), + telemetryService: null, + logger: Mock.Of())); + + Assert.Equal("telemetryService", exception.ParamName); + } + + [Fact] + public void Constructor_WhenLoggerIsNull_Throws() + { + var exception = Assert.Throws( + () => new DnxMaker( + storageFactory: Mock.Of(), + telemetryService: Mock.Of(), + logger: null)); + + Assert.Equal("logger", exception.ParamName); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public void GetRelativeAddressNupkg_WhenIdIsNullOrEmpty_Throws(string id) + { + var exception = Assert.Throws( + () => DnxMaker.GetRelativeAddressNupkg(id, version: "1.0.0")); + + Assert.Equal("id", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public void GetRelativeAddressNupkg_WhenVersionIsNullOrEmpty_Throws(string version) + { + var exception = Assert.Throws( + () => DnxMaker.GetRelativeAddressNupkg(id: "a", version: version)); + + Assert.Equal("version", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Fact] + public async Task HasPackageInIndexAsync_WhenStorageIsNull_Throws() + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.HasPackageInIndexAsync( + storage: null, + id: "a", + version: "b", + cancellationToken: CancellationToken.None)); + + Assert.Equal("storage", exception.ParamName); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task HasPackageInIndexAsync_WhenIdIsNullOrEmpty_Throws(string id) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.HasPackageInIndexAsync( + new MemoryStorage(), + id: id, + version: "a", + cancellationToken: CancellationToken.None)); + + Assert.Equal("id", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task HasPackageInIndexAsync_WhenVersionIsNullOrEmpty_Throws(string version) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.HasPackageInIndexAsync( + new MemoryStorage(), + id: "a", + version: version, + cancellationToken: CancellationToken.None)); + + Assert.Equal("version", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Fact] + public async Task HasPackageInIndexAsync_WhenCancelled_Throws() + { + var maker = CreateDnxMaker(); + + await Assert.ThrowsAsync( + () => maker.HasPackageInIndexAsync( + new MemoryStorage(), + id: "a", + version: "b", + cancellationToken: new CancellationToken(canceled: true))); + } + + [Fact] + public async Task HasPackageInIndexAsync_WhenPackageIdAndVersionDoNotExist_ReturnsFalse() + { + var catalogToDnxStorage = new MemoryStorage(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + var maker = new DnxMaker(catalogToDnxStorageFactory, Mock.Of(), Mock.Of()); + var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); + + var hasPackageInIndex = await maker.HasPackageInIndexAsync(storageForPackage, _packageId, "1.0.0", CancellationToken.None); + + Assert.False(hasPackageInIndex); + } + + [Fact] + public async Task HasPackageInIndexAsync_WhenPackageIdExistsButVersionDoesNotExist_ReturnsFalse() + { + var catalogToDnxStorage = new MemoryStorage(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + var maker = new DnxMaker(catalogToDnxStorageFactory, Mock.Of(), Mock.Of()); + + await maker.UpdatePackageVersionIndexAsync(_packageId, v => v.Add(NuGetVersion.Parse("1.0.0")), CancellationToken.None); + + var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); + + var hasPackageInIndex = await maker.HasPackageInIndexAsync(storageForPackage, _packageId, "2.0.0", CancellationToken.None); + + Assert.False(hasPackageInIndex); + } + + [Fact] + public async Task HasPackageInIndexAsync_WhenPackageIdAndVersionExist_ReturnsTrue() + { + var catalogToDnxStorage = new MemoryStorage(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + var maker = new DnxMaker(catalogToDnxStorageFactory, Mock.Of(), Mock.Of()); + + const string version = "1.0.0"; + + await maker.UpdatePackageVersionIndexAsync(_packageId, v => v.Add(NuGetVersion.Parse(version)), CancellationToken.None); + + var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); + + var hasPackageInIndex = await maker.HasPackageInIndexAsync(storageForPackage, _packageId, version, CancellationToken.None); + + Assert.True(hasPackageInIndex); + } + + [Fact] + public async Task AddPackageAsync_WhenNupkgStreamIsNull_Throws() + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + nupkgStream: null, + nuspec: "a", + packageId: "b", + normalizedPackageVersion: "c", + iconFilename: null, + cancellationToken: CancellationToken.None)); + + Assert.Equal("nupkgStream", exception.ParamName); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task AddPackageAsync_WhenNuspecIsNullOrEmpty_Throws(string nuspec) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Stream.Null, + nuspec, + packageId: "a", + normalizedPackageVersion: "b", + iconFilename: null, + cancellationToken: CancellationToken.None)); + + Assert.Equal("nuspec", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task AddPackageAsync_WhenPackageIdIsNullOrEmpty_Throws(string packageId) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Stream.Null, + nuspec: "a", + packageId: packageId, + normalizedPackageVersion: "b", + iconFilename: null, + cancellationToken: CancellationToken.None)); + + Assert.Equal("packageId", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task AddPackageAsync_WhenNormalizedPackageVersionIsNullOrEmpty_Throws(string normalizedPackageVersion) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Stream.Null, + nuspec: "a", + packageId: "b", + normalizedPackageVersion: normalizedPackageVersion, + iconFilename: null, + cancellationToken: CancellationToken.None)); + + Assert.Equal("normalizedPackageVersion", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Fact] + public async Task AddPackageAsync_WhenCancelled_Throws() + { + var maker = CreateDnxMaker(); + + await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Stream.Null, + nuspec: "a", + packageId: "b", + normalizedPackageVersion: "c", + iconFilename: null, + cancellationToken: new CancellationToken(canceled: true))); + } + + [Theory] + [MemberData(nameof(PackageVersions))] + public async Task AddPackageAsync_WithValidVersion_PopulatesStorageWithNupkgAndNuspec(string version) + { + var catalogToDnxStorage = new MemoryStorage(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + var maker = new DnxMaker(catalogToDnxStorageFactory, Mock.Of(), Mock.Of()); + var normalizedVersion = NuGetVersionUtility.NormalizeVersion(version); + + using (var nupkgStream = CreateFakePackageStream(_nupkgData)) + { + var dnxEntry = await maker.AddPackageAsync(nupkgStream, _nuspecData, _packageId, version, null, CancellationToken.None); + + var expectedNuspec = new Uri($"{catalogToDnxStorage.BaseAddress}{_packageId}/{normalizedVersion}/{_packageId}.nuspec"); + var expectedNupkg = new Uri($"{catalogToDnxStorage.BaseAddress}{_packageId}/{normalizedVersion}/{_packageId}.{normalizedVersion}.nupkg"); + var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); + + Assert.Equal(expectedNuspec, dnxEntry.Nuspec); + Assert.Equal(expectedNupkg, dnxEntry.Nupkg); + Assert.Equal(2, catalogToDnxStorage.Content.Count); + Assert.Equal(2, storageForPackage.Content.Count); + + Verify(catalogToDnxStorage, expectedNupkg, _nupkgData, _expectedCacheControl, _expectedPackageContentType); + Verify(catalogToDnxStorage, expectedNuspec, _nuspecData, _expectedCacheControl, _expectedNuspecContentType); + Verify(storageForPackage, expectedNupkg, _nupkgData, _expectedCacheControl, _expectedPackageContentType); + Verify(storageForPackage, expectedNuspec, _nuspecData, _expectedCacheControl, _expectedNuspecContentType); + } + } + + [Theory] + [InlineData("ahgjghaa.png", "")] + [InlineData("sdfgd.jpg", "")] + [InlineData("csdfsd.jpeg", "")] + public async Task AddPackageAsync_WhenEmbeddedIconPresent_SavesIcon(string iconFilename, string expectedContentType) + { + const string version = "1.2.3"; + const string imageContent = "Test image data"; + var imageDataBuffer = Encoding.UTF8.GetBytes(imageContent); + + var catalogToDnxStorage = new MemoryStorage(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + var maker = new DnxMaker(catalogToDnxStorageFactory, Mock.Of(), Mock.Of()); + + using (var nupkgStream = await CreateNupkgStreamWithIcon(iconFilename, imageDataBuffer)) + { + await maker.AddPackageAsync(nupkgStream, _nuspecData, _packageId, version, iconFilename, CancellationToken.None); + + var expectedIconUrl = new Uri($"{catalogToDnxStorage.BaseAddress}{_packageId}/{version}/icon"); + Verify(catalogToDnxStorage, expectedIconUrl, imageContent, _expectedCacheControl, expectedContentType); + } + } + + [Fact] + public async Task AddPackageAsync_WithStorage_WhenSourceStorageIsNull_Throws() + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + sourceStorage: null, + nuspec: "a", + packageId: "b", + normalizedPackageVersion: "c", + iconFilename: null, + cancellationToken: CancellationToken.None)); + + Assert.Equal("sourceStorage", exception.ParamName); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task AddPackageAsync_WithStorage_WhenNuspecIsNullOrEmpty_Throws(string nuspec) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Mock.Of(), + nuspec, + packageId: "a", + normalizedPackageVersion: "b", + iconFilename: null, + cancellationToken: CancellationToken.None)); + + Assert.Equal("nuspec", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task AddPackageAsync_WithStorage_WhenPackageIdIsNullOrEmpty_Throws(string id) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Mock.Of(), + nuspec: "a", + packageId: id, + normalizedPackageVersion: "b", + iconFilename: null, + cancellationToken: CancellationToken.None)); + + Assert.Equal("packageId", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task AddPackageAsync_WithStorage_WhenNormalizedPackageVersionIsNullOrEmpty_Throws(string version) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Mock.Of(), + nuspec: "a", + packageId: "b", + normalizedPackageVersion: version, + iconFilename: null, + cancellationToken: CancellationToken.None)); + + Assert.Equal("normalizedPackageVersion", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Fact] + public async Task AddPackageAsync_WithStorage_WhenCancelled_Throws() + { + var maker = CreateDnxMaker(); + + await Assert.ThrowsAsync( + () => maker.AddPackageAsync( + Mock.Of(), + nuspec: "a", + packageId: "b", + normalizedPackageVersion: "c", + iconFilename: null, + cancellationToken: new CancellationToken(canceled: true))); + } + + [Theory] + [MemberData(nameof(PackageVersions))] + public async Task AddPackageAsync_WithStorage_WithIStorage_PopulatesStorageWithNupkgAndNuspec(string version) + { + var catalogToDnxStorage = new AzureStorageStub(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + var maker = new DnxMaker(catalogToDnxStorageFactory, Mock.Of(), Mock.Of()); + var normalizedVersion = NuGetVersionUtility.NormalizeVersion(version); + var sourceStorage = new AzureStorageStub(); + + var dnxEntry = await maker.AddPackageAsync( + sourceStorage, + _nuspecData, + _packageId, + normalizedVersion, + null, + CancellationToken.None); + + var expectedNuspecUri = new Uri($"{catalogToDnxStorage.BaseAddress}{_packageId}/{normalizedVersion}/{_packageId}.nuspec"); + var expectedNupkgUri = new Uri($"{catalogToDnxStorage.BaseAddress}{_packageId}/{normalizedVersion}/{_packageId}.{normalizedVersion}.nupkg"); + var expectedSourceUri = new Uri(sourceStorage.BaseAddress, $"{_packageId}.{normalizedVersion}.nupkg"); + var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); + + Assert.Equal(expectedNuspecUri, dnxEntry.Nuspec); + Assert.Equal(expectedNupkgUri, dnxEntry.Nupkg); + Assert.Equal(2, catalogToDnxStorage.Content.Count); + Assert.Equal(2, storageForPackage.Content.Count); + + Verify(catalogToDnxStorage, expectedNupkgUri, expectedSourceUri.AbsoluteUri, _expectedCacheControl, _expectedPackageContentType); + Verify(catalogToDnxStorage, expectedNuspecUri, _nuspecData, _expectedCacheControl, _expectedNuspecContentType); + Verify(storageForPackage, expectedNupkgUri, expectedSourceUri.AbsoluteUri, _expectedCacheControl, _expectedPackageContentType); + Verify(storageForPackage, expectedNuspecUri, _nuspecData, _expectedCacheControl, _expectedNuspecContentType); + } + + [Theory] + [InlineData("sdafs.png", "")] + [InlineData("hjy.jpg", "")] + [InlineData("vfdg.jpeg", "")] + public async Task AddPackageAsync_WithStorage_WhenEmbeddedIconPresent_SavesIcon(string iconFilename, string expectedContentType) + { + const string version = "1.2.3"; + const string imageContent = "Test image data"; + var imageDataBuffer = Encoding.UTF8.GetBytes(imageContent); + + var catalogToDnxStorage = new AzureStorageStub(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + var maker = new DnxMaker(catalogToDnxStorageFactory, Mock.Of(), Mock.Of()); + var sourceStorageMock = new Mock(); + using (var nupkgStream = await CreateNupkgStreamWithIcon(iconFilename, imageDataBuffer)) + { + var cloudBlobMock = new Mock(); + cloudBlobMock + .Setup(cb => cb.GetStreamAsync(It.IsAny())) + .ReturnsAsync(nupkgStream); + + sourceStorageMock + .Setup(ss => ss.GetCloudBlockBlobReferenceAsync(It.IsAny())) + .ReturnsAsync(cloudBlobMock.Object); + + await maker.AddPackageAsync(sourceStorageMock.Object, _nuspecData, _packageId, version, iconFilename, CancellationToken.None); + + var expectedIconUrl = new Uri($"{catalogToDnxStorage.BaseAddress}{_packageId}/{version}/icon"); + Verify(catalogToDnxStorage, expectedIconUrl, imageContent, _expectedCacheControl, expectedContentType); + } + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task DeletePackageAsync_WhenIdIsNullOrEmpty_Throws(string id) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.DeletePackageAsync( + id: id, + version: "a", + cancellationToken: CancellationToken.None)); + + Assert.Equal("id", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task DeletePackageAsync_WhenVersionIsNullOrEmpty_Throws(string version) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.DeletePackageAsync( + id: "a", + version: version, + cancellationToken: CancellationToken.None)); + + Assert.Equal("version", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Fact] + public async Task DeletePackageAsync_WhenCancelled_Throws() + { + var maker = CreateDnxMaker(); + + await Assert.ThrowsAsync( + () => maker.DeletePackageAsync( + id: "a", + version: "b", + cancellationToken: new CancellationToken(canceled: true))); + } + + [Theory] + [MemberData(nameof(PackageVersions))] + public async Task DeletePackageAsync_WithValidVersion_RemovesNupkgAndNuspecFromStorage(string version) + { + var catalogToDnxStorage = new MemoryStorage(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + var maker = new DnxMaker(catalogToDnxStorageFactory, Mock.Of(), Mock.Of()); + + using (var nupkgStream = CreateFakePackageStream(_nupkgData)) + { + var dnxEntry = await maker.AddPackageAsync(nupkgStream, _nuspecData, _packageId, version, null, CancellationToken.None); + + var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); + + Assert.Equal(2, catalogToDnxStorage.Content.Count); + Assert.Equal(2, storageForPackage.Content.Count); + + await maker.DeletePackageAsync(_packageId, version, CancellationToken.None); + + Assert.Empty(catalogToDnxStorage.Content); + Assert.Empty(storageForPackage.Content); + } + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public async Task UpdatePackageVersionIndexAsync_WhenIdIsNullOrEmpty_Throws(string id) + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.UpdatePackageVersionIndexAsync( + id: id, + updateAction: _ => { }, + cancellationToken: CancellationToken.None)); + + Assert.Equal("id", exception.ParamName); + Assert.StartsWith("The argument must not be null or empty.", exception.Message); + } + + [Fact] + public async Task UpdatePackageVersionIndexAsync_WhenVersionIsNullOrEmpty_Throws() + { + var maker = CreateDnxMaker(); + + var exception = await Assert.ThrowsAsync( + () => maker.UpdatePackageVersionIndexAsync( + id: "a", + updateAction: null, + cancellationToken: CancellationToken.None)); + + Assert.Equal("updateAction", exception.ParamName); + } + + [Fact] + public async Task UpdatePackageVersionIndexAsync_WhenCancelled_Throws() + { + var maker = CreateDnxMaker(); + + await Assert.ThrowsAsync( + () => maker.UpdatePackageVersionIndexAsync( + id: "a", + updateAction: _ => { }, + cancellationToken: new CancellationToken(canceled: true))); + } + + [Theory] + [MemberData(nameof(PackageVersions))] + public async Task UpdatePackageVersionIndexAsync_WithValidVersion_CreatesIndex(string version) + { + var catalogToDnxStorage = new MemoryStorage(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + var maker = new DnxMaker(catalogToDnxStorageFactory, Mock.Of(), Mock.Of()); + var normalizedVersion = NuGetVersionUtility.NormalizeVersion(version); + + await maker.UpdatePackageVersionIndexAsync(_packageId, v => v.Add(NuGetVersion.Parse(version)), CancellationToken.None); + + var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); + var indexJsonUri = new Uri(storageForPackage.BaseAddress, "index.json"); + var indexJson = await storageForPackage.LoadAsync(indexJsonUri, CancellationToken.None); + var indexObject = JObject.Parse(indexJson.GetContentString()); + var versions = indexObject["versions"].ToObject(); + var expectedContent = GetExpectedIndexJsonContent(normalizedVersion); + + Assert.Single(catalogToDnxStorage.Content); + Assert.Single(storageForPackage.Content); + + Verify(catalogToDnxStorage, indexJsonUri, expectedContent, _expectedPackageVersionIndexJsonCacheControl, _expectedPackageVersionIndexJsonContentType); + Verify(storageForPackage, indexJsonUri, expectedContent, _expectedPackageVersionIndexJsonCacheControl, _expectedPackageVersionIndexJsonContentType); + + Assert.Equal(new[] { normalizedVersion }, versions); + } + + [Fact] + public async Task UpdatePackageVersionIndexAsync_WhenLastVersionRemoved_RemovesIndex() + { + var version = NuGetVersion.Parse("1.0.0"); + var catalogToDnxStorage = new MemoryStorage(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + var maker = new DnxMaker(catalogToDnxStorageFactory, Mock.Of(), Mock.Of()); + + await maker.UpdatePackageVersionIndexAsync(_packageId, v => v.Add(version), CancellationToken.None); + + var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); + var indexJsonUri = new Uri(storageForPackage.BaseAddress, "index.json"); + var indexJson = await storageForPackage.LoadAsync(indexJsonUri, CancellationToken.None); + + Assert.NotNull(indexJson); + Assert.Single(catalogToDnxStorage.Content); + Assert.Single(storageForPackage.Content); + + await maker.UpdatePackageVersionIndexAsync(_packageId, v => v.Remove(version), CancellationToken.None); + + indexJson = await storageForPackage.LoadAsync(indexJsonUri, CancellationToken.None); + + Assert.Null(indexJson); + Assert.Empty(catalogToDnxStorage.Content); + Assert.Empty(storageForPackage.Content); + } + + [Fact] + public async Task UpdatePackageVersionIndexAsync_WithNoVersions_DoesNotCreateIndex() + { + var catalogToDnxStorage = new MemoryStorage(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + var maker = new DnxMaker(catalogToDnxStorageFactory, Mock.Of(), Mock.Of()); + + await maker.UpdatePackageVersionIndexAsync(_packageId, v => { }, CancellationToken.None); + + var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); + var indexJsonUri = new Uri(storageForPackage.BaseAddress, "index.json"); + var indexJson = await storageForPackage.LoadAsync(indexJsonUri, CancellationToken.None); + + Assert.Null(indexJson); + Assert.Empty(catalogToDnxStorage.Content); + Assert.Empty(storageForPackage.Content); + } + + [Fact] + public async Task UpdatePackageVersionIndexAsync_WithMultipleVersions_SortsVersion() + { + var unorderedVersions = new[] + { + NuGetVersion.Parse("3.0.0"), + NuGetVersion.Parse("1.1.0"), + NuGetVersion.Parse("1.0.0"), + NuGetVersion.Parse("1.0.1"), + NuGetVersion.Parse("2.0.0"), + NuGetVersion.Parse("1.0.2") + }; + var catalogToDnxStorage = new MemoryStorage(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + var maker = new DnxMaker(catalogToDnxStorageFactory, Mock.Of(), Mock.Of()); + + await maker.UpdatePackageVersionIndexAsync(_packageId, v => v.UnionWith(unorderedVersions), CancellationToken.None); + + var storageForPackage = (MemoryStorage)catalogToDnxStorageFactory.Create(_packageId); + var indexJsonUri = new Uri(storageForPackage.BaseAddress, "index.json"); + var indexJson = await storageForPackage.LoadAsync(indexJsonUri, CancellationToken.None); + var indexObject = JObject.Parse(indexJson.GetContentString()); + var versions = indexObject["versions"].ToObject(); + + Assert.Single(catalogToDnxStorage.Content); + Assert.Single(storageForPackage.Content); + Assert.Collection( + versions, + version => Assert.Equal(unorderedVersions[2].ToNormalizedString(), version), + version => Assert.Equal(unorderedVersions[3].ToNormalizedString(), version), + version => Assert.Equal(unorderedVersions[5].ToNormalizedString(), version), + version => Assert.Equal(unorderedVersions[1].ToNormalizedString(), version), + version => Assert.Equal(unorderedVersions[4].ToNormalizedString(), version), + version => Assert.Equal(unorderedVersions[0].ToNormalizedString(), version)); + } + + public static IEnumerable PackageVersions + { + get + { + // normalized versions + yield return new object[] { "1.2.0" }; + yield return new object[] { "0.1.2" }; + yield return new object[] { "1.2.3.4" }; + yield return new object[] { "1.2.3-beta1" }; + yield return new object[] { "1.2.3-beta.1" }; + + // non-normalized versions + yield return new object[] { "1.2" }; + yield return new object[] { "1.2.3.0" }; + yield return new object[] { "1.02.3" }; + } + } + + private static DnxMaker CreateDnxMaker() + { + var catalogToDnxStorage = new MemoryStorage(); + var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); + + return new DnxMaker(catalogToDnxStorageFactory, Mock.Of(), Mock.Of()); + } + + private static MemoryStream CreateFakePackageStream(string content) + { + var stream = new MemoryStream(); + + using (var writer = new StreamWriter(stream, new UTF8Encoding(), bufferSize: 4096, leaveOpen: true)) + { + writer.Write(content); + writer.Flush(); + } + + stream.Position = 0; + + return stream; + } + + private static string GetExpectedIndexJsonContent(string version) + { + return $"{{\r\n \"versions\": [\r\n \"{version}\"\r\n ]\r\n}}"; + } + + private static void Verify( + MemoryStorage storage, + Uri uri, + string expectedContent, + string expectedCacheControl, + string expectedContentType) + { + Assert.True(storage.Content.TryGetValue(uri, out var content)); + Assert.Equal(expectedCacheControl, content.CacheControl); + Assert.Equal(expectedContentType, content.ContentType); + + Assert.True(storage.ContentBytes.TryGetValue(uri, out var bytes)); + Assert.Equal(Encoding.UTF8.GetBytes(expectedContent), bytes); + + var isExpected = storage.BaseAddress != new Uri("http://tempuri.org/"); + + Assert.Equal(isExpected, storage.ListMock.TryGetValue(uri, out var list)); + + if (isExpected) + { + Assert.Equal(uri, list.Uri); + + var utc = DateTime.UtcNow; + Assert.NotNull(list.LastModifiedUtc); + Assert.InRange(list.LastModifiedUtc.Value, utc.AddMinutes(-1), utc); + } + } + + private static async Task CreateNupkgStreamWithIcon(string iconFilename, byte[] imageDataBuffer) + { + var nupkgStream = new MemoryStream(); + using (var archive = new ZipArchive(nupkgStream, ZipArchiveMode.Create, leaveOpen: true)) + { + var entry = archive.CreateEntry(iconFilename); + using (var entryStream = entry.Open()) + { + await entryStream.WriteAsync(imageDataBuffer, 0, imageDataBuffer.Length); + } + } + + nupkgStream.Seek(0, SeekOrigin.Begin); + return nupkgStream; + } + + private sealed class AzureStorageStub : MemoryStorage, IAzureStorage + { + internal AzureStorageStub() + { + } + + private AzureStorageStub( + Uri baseAddress, + ConcurrentDictionary content, + ConcurrentDictionary contentBytes) + : base(baseAddress, content, contentBytes) + { + } + + public override Storage WithName(string name) + { + return new AzureStorageStub( + new Uri(BaseAddress + name), + Content, + ContentBytes); + } + + public Task GetCloudBlockBlobReferenceAsync(Uri blobUri) + { + throw new NotImplementedException(); + } + + public Task GetCloudBlockBlobReferenceAsync(string name) + { + throw new NotImplementedException(); + } + + public Task HasPropertiesAsync(Uri blobUri, string contentType, string cacheControl) + { + throw new NotImplementedException(); + } + + protected override Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + var destinationMemoryStorage = (AzureStorageStub)destinationStorage; + + string cacheControl = null; + string contentType = null; + + destinationProperties?.TryGetValue(StorageConstants.CacheControl, out cacheControl); + destinationProperties?.TryGetValue(StorageConstants.ContentType, out contentType); + + destinationMemoryStorage.Content[destinationUri] = new StringStorageContent(sourceUri.AbsoluteUri, contentType, cacheControl); + destinationMemoryStorage.ContentBytes[destinationUri] = Encoding.UTF8.GetBytes(sourceUri.AbsoluteUri); + destinationMemoryStorage.ListMock[destinationUri] = new StorageListItem(destinationUri, DateTime.UtcNow); + + return Task.FromResult(0); + } + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Extensions/DateTimeExtensionsTests.cs b/tests/CatalogTests/Extensions/DateTimeExtensionsTests.cs new file mode 100644 index 000000000..aa4a299fa --- /dev/null +++ b/tests/CatalogTests/Extensions/DateTimeExtensionsTests.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Xunit; + +namespace CatalogTests.Extensions +{ + public class DateTimeExtensionsTests + { + public class TheForceUtcMethod + { + [Fact] + public void ConvertsLocalTimeToUtc() + { + var localTime = new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Local); + + var convertedTime = localTime.ForceUtc(); + + Assert.Equal(DateTimeKind.Utc, convertedTime.Kind); + } + + [Fact] + public void ConvertsUnspecifiedTimeToUtc() + { + var unspecifiedTime = new DateTime(2019, 1, 1, 0, 0, 0, DateTimeKind.Unspecified); + + var convertedTime = unspecifiedTime.ForceUtc(); + + Assert.Equal(DateTimeKind.Utc, convertedTime.Kind); + } + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Extensions/DbDataReaderExtensionsTests.cs b/tests/CatalogTests/Extensions/DbDataReaderExtensionsTests.cs new file mode 100644 index 000000000..bac61e013 --- /dev/null +++ b/tests/CatalogTests/Extensions/DbDataReaderExtensionsTests.cs @@ -0,0 +1,98 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Data.Common; +using Moq; +using Xunit; + +namespace CatalogTests.Extensions +{ + public class DbDataReaderExtensionsTests + { + public class TheReadNullableInt32Method + : TheReadColumnOrNullMethodTestContainer + { + public TheReadNullableInt32Method() + : base((r, c) => r.ReadInt32OrNull(c)) + { + } + + public override void ReturnsValueWhenColumnHasValue() + { + const int ordinal = 1; + const int exampleValue = 5; + var dataReaderMock = new Mock(MockBehavior.Strict); + dataReaderMock.Setup(m => m.GetOrdinal(ColumnName)).Returns(ordinal); + dataReaderMock.Setup(m => m.IsDBNull(ordinal)).Returns(false); + dataReaderMock.Setup(m => m.GetInt32(ordinal)).Returns(exampleValue); + + var actual = ActualMethodBeingTested(dataReaderMock.Object, ColumnName); + + Assert.Equal(exampleValue, actual); + } + } + + public class TheReadNullableStringMethod + : TheReadColumnOrNullMethodTestContainer + { + public TheReadNullableStringMethod() + : base((r, c) => r.ReadStringOrNull(c)) + { + } + + public override void ReturnsValueWhenColumnHasValue() + { + const int ordinal = 1; + const string exampleValue = "test"; + var dataReaderMock = new Mock(MockBehavior.Strict); + dataReaderMock.Setup(m => m.GetOrdinal(ColumnName)).Returns(ordinal); + dataReaderMock.Setup(m => m.IsDBNull(ordinal)).Returns(false); + dataReaderMock.Setup(m => m.GetString(ordinal)).Returns(exampleValue); + + var actual = ActualMethodBeingTested(dataReaderMock.Object, ColumnName); + + Assert.Equal(exampleValue, actual); + } + } + + public abstract class TheReadColumnOrNullMethodTestContainer + { + protected const string ColumnName = "ColumnName"; + + protected readonly Func ActualMethodBeingTested; + + public TheReadColumnOrNullMethodTestContainer(Func actualMethodBeingTested) + { + ActualMethodBeingTested = actualMethodBeingTested ?? throw new ArgumentNullException(nameof(actualMethodBeingTested)); + } + + [Fact] + public void ReturnsNullWhenColumnDoesNotExist() + { + var dataReaderMock = new Mock(MockBehavior.Strict); + dataReaderMock.Setup(m => m.GetOrdinal(ColumnName)).Throws(); + + var actual = ActualMethodBeingTested(dataReaderMock.Object, ColumnName); + + Assert.Null(actual); + } + + [Fact] + public abstract void ReturnsValueWhenColumnHasValue(); + + [Fact] + public void ReturnsNullWhenColumnHasNoValue() + { + const int ordinal = 1; + var dataReaderMock = new Mock(MockBehavior.Strict); + dataReaderMock.Setup(m => m.GetOrdinal(ColumnName)).Returns(ordinal); + dataReaderMock.Setup(m => m.IsDBNull(ordinal)).Returns(true); + + var actual = ActualMethodBeingTested(dataReaderMock.Object, ColumnName); + + Assert.Null(actual); + } + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/AsyncExtensionsTests.cs b/tests/CatalogTests/Helpers/AsyncExtensionsTests.cs new file mode 100644 index 000000000..10d81ca95 --- /dev/null +++ b/tests/CatalogTests/Helpers/AsyncExtensionsTests.cs @@ -0,0 +1,71 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Xunit; +using NuGetAsyncExtensions = NuGet.Services.Metadata.Catalog.Helpers.AsyncExtensions; + +namespace CatalogTests.Helpers +{ + public class AsyncExtensionsTests + { + [Fact] + public async Task ForEachAsync_WhenEnumerableIsNull_Throws() + { + IEnumerable enumerable = null; + + var exception = await Assert.ThrowsAsync( + () => NuGetAsyncExtensions.ForEachAsync(enumerable, maxDegreeOfParallelism: 2, func: _ => Task.FromResult(0))); + + Assert.Equal("enumerable", exception.ParamName); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public async Task ForEachAsync_WhenMaxDegreeOfParallelismIsLessThanOne_Throws(int maxDegreeOfParallelism) + { + var exception = await Assert.ThrowsAsync( + () => NuGetAsyncExtensions.ForEachAsync(Enumerable.Empty(), maxDegreeOfParallelism, func: _ => Task.FromResult(0))); + + Assert.Equal("maxDegreeOfParallelism", exception.ParamName); + Assert.StartsWith($"The argument must be within the range from 1 (inclusive) to {int.MaxValue} (inclusive).", exception.Message); + } + + [Fact] + public async Task ForEachAsync_WhenFuncIsNull_Throws() + { + var exception = await Assert.ThrowsAsync( + () => NuGetAsyncExtensions.ForEachAsync(Enumerable.Empty(), maxDegreeOfParallelism: 2, func: null)); + + Assert.Equal("func", exception.ParamName); + } + + [Fact] + public async Task ForEachAsync_WithValidArguments_ProcessesAllItems() + { + var enumerable = Enumerable.Range(1, 100); + var bag = new ConcurrentBag(); + + await NuGetAsyncExtensions.ForEachAsync( + enumerable, + maxDegreeOfParallelism: 10, + func: i => + { + bag.Add(i); + + return Task.FromResult(0); + }); + + var actualResults = bag.ToArray(); + + Array.Sort(actualResults); + + Assert.Equal(enumerable, actualResults); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/CatalogIndependentPackageDetails.cs b/tests/CatalogTests/Helpers/CatalogIndependentPackageDetails.cs new file mode 100644 index 000000000..93dce248e --- /dev/null +++ b/tests/CatalogTests/Helpers/CatalogIndependentPackageDetails.cs @@ -0,0 +1,214 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Security.Cryptography; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests; +using NgTests.Infrastructure; + +namespace CatalogTests.Helpers +{ + internal sealed class CatalogIndependentPackageDetails + { + private static readonly JObject _context = JObject.Parse( +@"{ + ""@vocab"": ""http://schema.nuget.org/schema#"", + ""catalog"": ""http://schema.nuget.org/catalog#"", + ""xsd"": ""http://www.w3.org/2001/XMLSchema#"", + ""dependencies"": { + ""@id"": ""dependency"", + ""@container"": ""@set"" + }, + ""dependencyGroups"": { + ""@id"": ""dependencyGroup"", + ""@container"": ""@set"" + }, + ""packageEntries"": { + ""@id"": ""packageEntry"", + ""@container"": ""@set"" + }, + ""packageTypes"": { + ""@id"": ""packageType"", + ""@container"": ""@set"" + }, + ""supportedFrameworks"": { + ""@id"": ""supportedFramework"", + ""@container"": ""@set"" + }, + ""tags"": { + ""@id"": ""tag"", + ""@container"": ""@set"" + }, + ""published"": { + ""@type"": ""xsd:dateTime"" + }, + ""created"": { + ""@type"": ""xsd:dateTime"" + }, + ""lastEdited"": { + ""@type"": ""xsd:dateTime"" + }, + ""catalog:commitTimeStamp"": { + ""@type"": ""xsd:dateTime"" + } + }"); + + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string[] TypeKeyword { get; } + [JsonProperty(CatalogConstants.Authors)] + internal string Authors { get; } + [JsonProperty(CatalogConstants.CatalogCommitId)] + internal string CommitId { get; } + [JsonProperty(CatalogConstants.CatalogCommitTimeStamp)] + internal string CommitTimeStamp { get; } + [JsonProperty(CatalogConstants.Created)] + internal string Created { get; } + [JsonProperty(CatalogConstants.Deprecation)] + internal RegistrationPackageDeprecation Deprecation { get; } + [JsonProperty(CatalogConstants.Description)] + internal string Description { get; } + [JsonProperty(CatalogConstants.Id)] + internal string Id { get; } + [JsonProperty(CatalogConstants.IsPrerelease)] + internal bool IsPrerelease { get; } + [JsonProperty(CatalogConstants.LastEdited)] + internal string LastEdited { get; } + [JsonProperty(CatalogConstants.Listed)] + internal bool Listed { get; } + [JsonProperty(CatalogConstants.PackageHash)] + internal string PackageHash { get; } + [JsonProperty(CatalogConstants.PackageHashAlgorithm)] + internal string PackageHashAlgorithm { get; } + [JsonProperty(CatalogConstants.PackageSize)] + internal int PackageSize { get; } + [JsonProperty(CatalogConstants.Published)] + internal string Published { get; } + [JsonProperty(CatalogConstants.RequireLicenseAcceptance)] + internal bool RequireLicenseAcceptance { get; } + [JsonProperty(CatalogConstants.VerbatimVersion)] + internal string VerbatimVersion { get; } + [JsonProperty(CatalogConstants.Version)] + internal string Version { get; } + [JsonProperty(CatalogConstants.PackageEntries)] + internal CatalogPackageEntry[] PackageEntries { get; } + [JsonProperty(CatalogConstants.ContextKeyword)] + internal JObject ContextKeyword { get; } + + internal CatalogIndependentPackageDetails( + string id = null, + string version = null, + string baseUri = null, + string commitId = null, + DateTimeOffset? commitTimeStamp = null, + RegistrationPackageDeprecation deprecation = null) + { + var utc = commitTimeStamp ?? DateTimeOffset.UtcNow; + + Id = id ?? TestUtility.CreateRandomAlphanumericString(); + + var build = (int)(utc.Ticks & 0xff); // random number + + VerbatimVersion = version ?? $"1.0.{build}"; + Version = version ?? VerbatimVersion; + + IdKeyword = $"{baseUri ?? "https://nuget.test/"}" + + $"v3-catalog0/data/{utc.ToString(CatalogConstants.UrlTimeStampFormat)}" + + $"/{Id.ToLowerInvariant()}.{Version.ToLowerInvariant()}.json"; + TypeKeyword = new[] { CatalogConstants.PackageDetails, CatalogConstants.CatalogPermalink }; + Authors = TestUtility.CreateRandomAlphanumericString(); + CommitId = commitId ?? Guid.NewGuid().ToString("D"); + CommitTimeStamp = utc.ToString(CatalogConstants.CommitTimeStampFormat); + Created = utc.AddHours(-2).ToString(CatalogConstants.DateTimeFormat); + Deprecation = deprecation; + Description = TestUtility.CreateRandomAlphanumericString(); + LastEdited = utc.AddHours(-1).ToString(CatalogConstants.DateTimeFormat); + Listed = true; + PackageHash = CreateFakePackageHash(); + PackageHashAlgorithm = TestUtility.CreateRandomAlphanumericString(); + PackageSize = (int)(utc.Ticks & 0xffffff); // random number + Published = Created; + RequireLicenseAcceptance = utc.Ticks % 2 == 0; + + PackageEntries = new[] + { + new CatalogPackageEntry( + idKeyword: $"{IdKeyword}#{Id}.nuspec", + typeKeyword: CatalogConstants.PackageEntry, + compressedLength: (int)(utc.Ticks & 0xffff) + 1, + fullName: $"{Id}.nuspec", + length: (int)(utc.Ticks & 0xfff) + 1, + name: $"{Id}.nuspec"), + new CatalogPackageEntry( + idKeyword: $"{IdKeyword}#.signature.p7s", + typeKeyword: CatalogConstants.PackageEntry, + compressedLength: (int)(utc.Ticks & 0xffff), + fullName: ".signature.p7s", + length: (int)(utc.Ticks & 0xfff), + name: ".signature.p7s") + }; + + ContextKeyword = _context; + } + + [JsonConstructor] + internal CatalogIndependentPackageDetails( + string idKeyword, + string[] typeKeyword, + string authors, + string commitId, + string commitTimeStamp, + string created, + string description, + string id, + bool isPrerelease, + string lastEdited, + bool listed, + string packageHash, + string packageHashAlgorithm, + int packageSize, + string published, + bool requireLicenseAcceptance, + string verbatimVersion, + string version, + CatalogPackageEntry[] packageEntries, + JObject contextKeyword) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + Authors = authors; + CommitId = commitId; + CommitTimeStamp = commitTimeStamp; + Created = created; + Description = description; + Id = id; + IsPrerelease = isPrerelease; + LastEdited = lastEdited; + Listed = listed; + PackageHash = packageHash; + PackageHashAlgorithm = packageHashAlgorithm; + PackageSize = packageSize; + Published = published; + RequireLicenseAcceptance = requireLicenseAcceptance; + VerbatimVersion = verbatimVersion; + Version = version; + PackageEntries = packageEntries; + ContextKeyword = contextKeyword; + } + + private static string CreateFakePackageHash() + { + using (var rng = RandomNumberGenerator.Create()) + { + var bytes = new byte[64]; + + rng.GetBytes(bytes); + + return Convert.ToBase64String(bytes); + } + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/CatalogIndependentPage.cs b/tests/CatalogTests/Helpers/CatalogIndependentPage.cs new file mode 100644 index 000000000..abf632bc5 --- /dev/null +++ b/tests/CatalogTests/Helpers/CatalogIndependentPage.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class CatalogIndependentPage : CatalogPage + { + [JsonProperty(CatalogConstants.Parent)] + internal string Parent { get; } + [JsonProperty(CatalogConstants.Items)] + internal CatalogPackageDetails[] Items { get; } + [JsonProperty(CatalogConstants.ContextKeyword)] + internal JObject ContextKeyword { get; } + + [JsonConstructor] + internal CatalogIndependentPage( + string idKeyword, + string typeKeyword, + string commitId, + string commitTimeStamp, + int count, + string parent, + CatalogPackageDetails[] items, + JObject contextKeyword) + : base(idKeyword, typeKeyword, commitId, commitTimeStamp, count) + { + Parent = parent; + Items = items; + ContextKeyword = contextKeyword; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/CatalogIndex.cs b/tests/CatalogTests/Helpers/CatalogIndex.cs new file mode 100644 index 000000000..5142adce6 --- /dev/null +++ b/tests/CatalogTests/Helpers/CatalogIndex.cs @@ -0,0 +1,75 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class CatalogIndex + { + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string[] TypeKeyword { get; } + [JsonProperty(CatalogConstants.CommitId)] + internal string CommitId { get; } + [JsonProperty(CatalogConstants.CommitTimeStamp)] + internal string CommitTimeStamp { get; } + [JsonProperty(CatalogConstants.Count)] + internal int Count { get; } + [JsonProperty(CatalogConstants.NuGetLastCreated)] + internal string LastCreated { get; } + [JsonProperty(CatalogConstants.NuGetLastDeleted)] + internal string LastDeleted { get; } + [JsonProperty(CatalogConstants.NuGetLastEdited)] + internal string LastEdited { get; } + [JsonProperty(CatalogConstants.Items)] + internal CatalogPage[] Items { get; } + [JsonProperty(CatalogConstants.ContextKeyword)] + internal JObject ContextKeyword { get; } + + [JsonConstructor] + internal CatalogIndex( + string idKeyword, + string[] typeKeyword, + string commitId, + string commitTimeStamp, + int count, + string lastCreated, + string lastDeleted, + string lastEdited, + CatalogPage[] items, + JObject contextKeyword) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + CommitId = commitId; + CommitTimeStamp = commitTimeStamp; + Count = count; + Items = items; + ContextKeyword = contextKeyword; + } + + internal static CatalogIndex Create(CatalogIndependentPage page, JObject contextKeyword) + { + var lastCreated = page.CommitTimeStamp; + var lastDeleted = page.CommitTimeStamp; + var lastEdited = page.CommitTimeStamp; + var pages = new[] { CatalogPage.Create(page) }; + + return new CatalogIndex( + page.Parent, + new[] { CatalogConstants.CatalogRoot, CatalogConstants.AppendOnlyCatalog, CatalogConstants.Permalink }, + page.CommitId, + page.CommitTimeStamp, + pages.Length, + lastCreated, + lastDeleted, + lastEdited, + pages, + contextKeyword); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/CatalogPackageDetails.cs b/tests/CatalogTests/Helpers/CatalogPackageDetails.cs new file mode 100644 index 000000000..d7abc1eef --- /dev/null +++ b/tests/CatalogTests/Helpers/CatalogPackageDetails.cs @@ -0,0 +1,52 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class CatalogPackageDetails + { + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string TypeKeyword { get; } + [JsonProperty(CatalogConstants.CommitId)] + internal string CommitId { get; } + [JsonProperty(CatalogConstants.CommitTimeStamp)] + internal string CommitTimeStamp { get; } + [JsonProperty(CatalogConstants.NuGetId)] + internal string Id { get; } + [JsonProperty(CatalogConstants.NuGetVersion)] + internal string Version { get; } + + [JsonConstructor] + internal CatalogPackageDetails( + string idKeyword, + string typeKeyword, + string commitId, + string commitTimeStamp, + string id, + string version) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + CommitId = commitId; + CommitTimeStamp = commitTimeStamp; + Id = id; + Version = version; + } + + internal static CatalogPackageDetails Create(CatalogIndependentPackageDetails details) + { + return new CatalogPackageDetails( + details.IdKeyword, + CatalogConstants.NuGetPackageDetails, + details.CommitId, + details.CommitTimeStamp, + details.Id, + details.Version); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/CatalogPackageEntry.cs b/tests/CatalogTests/Helpers/CatalogPackageEntry.cs new file mode 100644 index 000000000..b14c5eca5 --- /dev/null +++ b/tests/CatalogTests/Helpers/CatalogPackageEntry.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class CatalogPackageEntry + { + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string TypeKeyword { get; } + [JsonProperty(CatalogConstants.CompressedLength)] + internal long CompressedLength { get; } + [JsonProperty(CatalogConstants.FullName)] + internal string FullName { get; } + [JsonProperty(CatalogConstants.Length)] + internal long Length { get; } + [JsonProperty(CatalogConstants.Name)] + internal string Name { get; } + + [JsonConstructor] + internal CatalogPackageEntry( + string idKeyword, + string typeKeyword, + long compressedLength, + string fullName, + long length, + string name) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + CompressedLength = compressedLength; + FullName = fullName; + Length = length; + Name = name; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/CatalogPage.cs b/tests/CatalogTests/Helpers/CatalogPage.cs new file mode 100644 index 000000000..18c16a477 --- /dev/null +++ b/tests/CatalogTests/Helpers/CatalogPage.cs @@ -0,0 +1,47 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal class CatalogPage + { + [JsonProperty(CatalogConstants.IdKeyword)] + public string IdKeyword { get; protected set; } + [JsonProperty(CatalogConstants.TypeKeyword)] + public string TypeKeyword { get; protected set; } + [JsonProperty(CatalogConstants.CommitId)] + public string CommitId { get; } + [JsonProperty(CatalogConstants.CommitTimeStamp)] + public string CommitTimeStamp { get; protected set; } + [JsonProperty(CatalogConstants.Count)] + public int Count { get; protected set; } + + [JsonConstructor] + internal CatalogPage( + string idKeyword, + string typeKeyword, + string commitId, + string commitTimeStamp, + int count) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + CommitId = commitId; + CommitTimeStamp = commitTimeStamp; + Count = count; + } + + internal static CatalogPage Create(CatalogIndependentPage page) + { + return new CatalogPage( + page.IdKeyword, + page.TypeKeyword, + page.CommitId, + page.CommitTimeStamp, + page.Count); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/CatalogPropertiesTests.cs b/tests/CatalogTests/Helpers/CatalogPropertiesTests.cs new file mode 100644 index 000000000..efe15f318 --- /dev/null +++ b/tests/CatalogTests/Helpers/CatalogPropertiesTests.cs @@ -0,0 +1,233 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Moq; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests.Helpers +{ + public class CatalogPropertiesTests + { + [Fact] + public void Constructor_InitializesPropertiesWithNull() + { + var properties = new CatalogProperties(lastCreated: null, lastDeleted: null, lastEdited: null); + + Assert.Null(properties.LastCreated); + Assert.Null(properties.LastDeleted); + Assert.Null(properties.LastEdited); + } + + [Fact] + public void Constructor_InitializesPropertiesWithNonNullValues() + { + var lastCreated = DateTime.Now; + var lastDeleted = lastCreated.AddMinutes(1); + var lastEdited = lastDeleted.AddMinutes(1); + var properties = new CatalogProperties(lastCreated, lastDeleted, lastEdited); + + Assert.Equal(lastCreated, properties.LastCreated); + Assert.Equal(lastDeleted, properties.LastDeleted); + Assert.Equal(lastEdited, properties.LastEdited); + } + + [Fact] + public async Task ReadAsync_ThrowsForNullStorage() + { + var exception = await Assert.ThrowsAsync( + () => CatalogProperties.ReadAsync( + storage: null, + telemetryService: Mock.Of(), + cancellationToken: CancellationToken.None)); + + Assert.Equal("storage", exception.ParamName); + } + + [Fact] + public async Task ReadAsync_ThrowsForNullTelemetryService() + { + var exception = await Assert.ThrowsAsync( + () => CatalogProperties.ReadAsync( + storage: Mock.Of(), + telemetryService: null, + cancellationToken: CancellationToken.None)); + + Assert.Equal("telemetryService", exception.ParamName); + } + + [Fact] + public async Task ReadAsync_ThrowsIfCancelled() + { + await Assert.ThrowsAsync( + () => CatalogProperties.ReadAsync( + Mock.Of(), + Mock.Of(), + new CancellationToken(canceled: true))); + } + + [Fact] + public async Task ReadAsync_ReturnsNullPropertiesIfStorageReturnsNull() + { + var storage = CreateStorageMock(json: null); + + var catalogProperties = await CatalogProperties.ReadAsync( + storage.Object, + Mock.Of(), + CancellationToken.None); + + Assert.NotNull(catalogProperties); + Assert.Null(catalogProperties.LastCreated); + Assert.Null(catalogProperties.LastDeleted); + Assert.Null(catalogProperties.LastEdited); + + storage.Verify(); + } + + [Fact] + public async Task ReadAsync_ReturnsNullPropertiesIfPropertiesMissing() + { + var storage = CreateStorageMock(json: "{}"); + + var catalogProperties = await CatalogProperties.ReadAsync( + storage.Object, + Mock.Of(), + CancellationToken.None); + + Assert.NotNull(catalogProperties); + Assert.Null(catalogProperties.LastCreated); + Assert.Null(catalogProperties.LastDeleted); + Assert.Null(catalogProperties.LastEdited); + + storage.Verify(); + } + + [Fact] + public async Task ReadAsync_ReturnsAllPropertiesIfAllPropertiesSet() + { + var lastCreatedDatetimeOffset = CreateDateTimeOffset(TimeSpan.FromMinutes(-5)); + var lastDeletedDatetimeOffset = CreateDateTimeOffset(TimeSpan.Zero); + var lastEditedDatetimeOffset = CreateDateTimeOffset(TimeSpan.FromMinutes(-3)); + var json = $"{{\"nuget:lastCreated\":\"{(lastCreatedDatetimeOffset.ToString("O"))}\"," + + $"\"nuget:lastDeleted\":\"{(lastDeletedDatetimeOffset.ToString("O"))}\"," + + $"\"nuget:lastEdited\":\"{(lastEditedDatetimeOffset.ToString("O"))}\"}}"; + var storage = CreateStorageMock(json); + + var catalogProperties = await CatalogProperties.ReadAsync( + storage.Object, + Mock.Of(), + CancellationToken.None); + + Assert.NotNull(catalogProperties); + Assert.NotNull(catalogProperties.LastCreated); + Assert.NotNull(catalogProperties.LastDeleted); + Assert.NotNull(catalogProperties.LastEdited); + Assert.Equal(lastCreatedDatetimeOffset, catalogProperties.LastCreated.Value); + Assert.Equal(lastDeletedDatetimeOffset, catalogProperties.LastDeleted.Value); + Assert.Equal(lastEditedDatetimeOffset, catalogProperties.LastEdited.Value); + + storage.Verify(); + } + + [Fact] + public async Task ReadAsync_ReturnsDateTimeWithFractionalHourOffsetInUtc() + { + var lastCreated = CreateDateTimeOffset(new TimeSpan(hours: 5, minutes: 30, seconds: 0)); + var json = CreateCatalogJson("nuget:lastCreated", lastCreated); + + await VerifyPropertyAsync(json, catalogProperties => + { + Assert.NotNull(catalogProperties.LastCreated); + Assert.Equal(lastCreated, catalogProperties.LastCreated.Value); + Assert.Equal(DateTimeKind.Utc, catalogProperties.LastCreated.Value.Kind); + }); + } + + [Fact] + public async Task ReadAsync_ReturnsDateTimeWithPositiveOffsetInUtc() + { + var lastDeleted = CreateDateTimeOffset(TimeSpan.FromHours(1)); + var json = CreateCatalogJson("nuget:lastDeleted", lastDeleted); + + await VerifyPropertyAsync(json, catalogProperties => + { + Assert.NotNull(catalogProperties.LastDeleted); + Assert.Equal(lastDeleted, catalogProperties.LastDeleted.Value); + Assert.Equal(DateTimeKind.Utc, catalogProperties.LastDeleted.Value.Kind); + }); + } + + [Fact] + public async Task ReadAsync_ReturnsDateTimeWithNegativeOffsetInUtc() + { + var lastEdited = CreateDateTimeOffset(TimeSpan.FromHours(-1)); + var json = CreateCatalogJson("nuget:lastEdited", lastEdited); + + await VerifyPropertyAsync(json, catalogProperties => + { + Assert.NotNull(catalogProperties.LastEdited); + Assert.Equal(lastEdited, catalogProperties.LastEdited.Value); + Assert.Equal(DateTimeKind.Utc, catalogProperties.LastEdited.Value.Kind); + }); + } + + [Fact] + public async Task ReadAsync_ReturnUtcDateTimeInUtc() + { + var lastCreated = DateTimeOffset.UtcNow; + var json = CreateCatalogJson("nuget:lastCreated", lastCreated); + + await VerifyPropertyAsync(json, catalogProperties => + { + Assert.NotNull(catalogProperties.LastCreated); + Assert.Equal(lastCreated, catalogProperties.LastCreated.Value); + Assert.Equal(DateTimeKind.Utc, catalogProperties.LastCreated.Value.Kind); + }); + } + + private static string CreateCatalogJson(string propertyName, DateTimeOffset propertyValue) + { + return $"{{\"{propertyName}\":\"{(propertyValue.ToString("O"))}\"}}"; + } + + private static DateTimeOffset CreateDateTimeOffset(TimeSpan offset) + { + var datetime = new DateTime(DateTime.Now.Ticks, DateTimeKind.Unspecified); + + return new DateTimeOffset(datetime, offset); + } + + private static async Task VerifyPropertyAsync(string json, Action propertyVerifier) + { + var storage = CreateStorageMock(json); + + var catalogProperties = await CatalogProperties.ReadAsync( + storage.Object, + Mock.Of(), + CancellationToken.None); + + Assert.NotNull(catalogProperties); + propertyVerifier(catalogProperties); + storage.Verify(); + } + + private static Mock CreateStorageMock(string json) + { + var storage = new Mock(MockBehavior.Strict); + + storage.Setup(x => x.ResolveUri(It.IsNotNull())) + .Returns(new Uri("https://unit.test")) + .Verifiable(); + storage.Setup(x => x.LoadStringAsync(It.IsNotNull(), It.IsAny())) + .ReturnsAsync(json) + .Verifiable(); + + return storage; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/CatalogWriterHelperTests.cs b/tests/CatalogTests/Helpers/CatalogWriterHelperTests.cs new file mode 100644 index 000000000..c9a497f2f --- /dev/null +++ b/tests/CatalogTests/Helpers/CatalogWriterHelperTests.cs @@ -0,0 +1,527 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using Newtonsoft.Json.Linq; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests.Helpers +{ + public class CatalogWriterHelperTests + { + public class TheWritePackageDetailsToCatalogAsyncMethod + { + [Fact] + public async Task WhenPackageCatalogItemCreatorIsNull_Throws() + { + IPackageCatalogItemCreator creator = null; + + var exception = await Assert.ThrowsAsync( + () => CatalogWriterHelper.WritePackageDetailsToCatalogAsync( + creator, + new SortedList>(), + Mock.Of(), + DateTime.MinValue, + DateTime.MinValue, + DateTime.MinValue, + maxDegreeOfParallelism: 1, + createdPackages: null, + updateCreatedFromEdited: false, + cancellationToken: CancellationToken.None, + telemetryService: Mock.Of(), + logger: Mock.Of())); + + Assert.Equal("packageCatalogItemCreator", exception.ParamName); + } + + [Fact] + public async Task WhenPackagesIsNull_Throws() + { + SortedList> packages = null; + + var exception = await Assert.ThrowsAsync( + () => CatalogWriterHelper.WritePackageDetailsToCatalogAsync( + Mock.Of(), + packages, + Mock.Of(), + DateTime.MinValue, + DateTime.MinValue, + DateTime.MinValue, + maxDegreeOfParallelism: 1, + createdPackages: null, + updateCreatedFromEdited: false, + cancellationToken: CancellationToken.None, + telemetryService: Mock.Of(), + logger: Mock.Of())); + + Assert.Equal("packages", exception.ParamName); + } + + [Fact] + public async Task WhenStorageIsNull_Throws() + { + IStorage storage = null; + + var exception = await Assert.ThrowsAsync( + () => CatalogWriterHelper.WritePackageDetailsToCatalogAsync( + Mock.Of(), + new SortedList>(), + storage, + DateTime.MinValue, + DateTime.MinValue, + DateTime.MinValue, + maxDegreeOfParallelism: 1, + createdPackages: null, + updateCreatedFromEdited: false, + cancellationToken: CancellationToken.None, + telemetryService: Mock.Of(), + logger: Mock.Of())); + + Assert.Equal("storage", exception.ParamName); + } + + [Theory] + [InlineData(-1)] + [InlineData(0)] + public async Task WhenMaxDegreeOfParallelismIsOutOfRange_Throws(int maxDegreeOfParallelism) + { + var exception = await Assert.ThrowsAsync( + () => CatalogWriterHelper.WritePackageDetailsToCatalogAsync( + Mock.Of(), + new SortedList>(), + Mock.Of(), + DateTime.UtcNow, + DateTime.UtcNow, + DateTime.UtcNow, + maxDegreeOfParallelism, + createdPackages: false, + updateCreatedFromEdited: false, + cancellationToken: CancellationToken.None, + telemetryService: Mock.Of(), + logger: Mock.Of())); + + Assert.Equal("maxDegreeOfParallelism", exception.ParamName); + Assert.StartsWith($"The argument must be within the range from 1 (inclusive) to {int.MaxValue} (inclusive).", exception.Message); + } + + [Fact] + public async Task WhenTelemetryServiceIsNull_Throws() + { + var exception = await Assert.ThrowsAsync( + () => CatalogWriterHelper.WritePackageDetailsToCatalogAsync( + Mock.Of(), + new SortedList>(), + Mock.Of(), + DateTime.MinValue, + DateTime.MinValue, + DateTime.MinValue, + maxDegreeOfParallelism: 1, + createdPackages: null, + updateCreatedFromEdited: false, + cancellationToken: CancellationToken.None, + telemetryService: null, + logger: Mock.Of())); + + Assert.Equal("telemetryService", exception.ParamName); + } + + [Fact] + public async Task WhenLoggerIsNull_Throws() + { + var exception = await Assert.ThrowsAsync( + () => CatalogWriterHelper.WritePackageDetailsToCatalogAsync( + Mock.Of(), + new SortedList>(), + Mock.Of(), + DateTime.MinValue, + DateTime.MinValue, + DateTime.MinValue, + maxDegreeOfParallelism: 1, + createdPackages: null, + updateCreatedFromEdited: false, + cancellationToken: CancellationToken.None, + telemetryService: Mock.Of(), + logger: null)); + + Assert.Equal("logger", exception.ParamName); + } + + [Fact] + public async Task WhenCancellationTokenIsCancelled_Throws() + { + await Assert.ThrowsAsync( + () => CatalogWriterHelper.WritePackageDetailsToCatalogAsync( + Mock.Of(), + new SortedList>(), + Mock.Of(), + DateTime.MinValue, + DateTime.MinValue, + DateTime.MinValue, + maxDegreeOfParallelism: 1, + createdPackages: null, + updateCreatedFromEdited: false, + cancellationToken: new CancellationToken(canceled: true), + telemetryService: Mock.Of(), + logger: Mock.Of())); + } + + [Fact] + public async Task WhenCreatedPackagesIsNull_WithNoPackages_ReturnsDateTimeMinValue() + { + using (var test = new WritePackageDetailsToCatalogAsyncTest()) + { + test.CreatedPackages = null; + + var result = await test.WritePackageDetailsToCatalogAsync(); + + Assert.Equal(DateTime.MinValue, result); + } + } + + [Fact] + public async Task WhenCreatedPackagesIsFalse_WithNoPackages_ReturnsLastEdited() + { + using (var test = new WritePackageDetailsToCatalogAsyncTest()) + { + test.CreatedPackages = false; + + var result = await test.WritePackageDetailsToCatalogAsync(); + + Assert.Equal(test.LastEdited, result); + } + } + + [Fact] + public async Task WhenCreatedPackagesIsTrue_WithNoPackages_ReturnsLastCreated() + { + using (var test = new WritePackageDetailsToCatalogAsyncTest()) + { + test.CreatedPackages = true; + + var result = await test.WritePackageDetailsToCatalogAsync(); + + Assert.Equal(test.LastCreated, result); + } + } + + public static IEnumerable WithOnePackage_UpdatesStorage_Data + { + get + { + foreach (var createdPackages in new[] { false, true }) + { + foreach (var updateCreatedFromEdited in new[] { false, true }) + { + foreach (var deprecationState in Enum.GetValues(typeof(PackageDeprecationItemState)).Cast()) + { + for (var vulnerabilityCount = 0; vulnerabilityCount < 3; vulnerabilityCount++) + { + yield return new object[] { createdPackages, updateCreatedFromEdited, deprecationState, vulnerabilityCount }; + } + } + } + } + } + } + + [Theory] + [MemberData(nameof(WithOnePackage_UpdatesStorage_Data))] + public async Task WithOnePackage_UpdatesStorage(bool createdPackages, bool updateCreatedFromEdited, PackageDeprecationItemState deprecationState, int vulnerabilityCount) + { + // Arrange + using (var test = new WritePackageDetailsToCatalogAsyncTest()) + { + test.CreatedPackages = createdPackages; + test.UpdateCreatedFromEdited = updateCreatedFromEdited; + test.Packages.Add(test.UtcNow, new List() + { + test.FeedPackageDetails + }); + + NupkgMetadata nupkgMetadata = GetNupkgMetadata("Newtonsoft.Json.9.0.2-beta1.nupkg"); + + var deprecationItem = GetPackageDeprecationItemFromState(deprecationState); + var vulnerabilities = CreateTestPackageVulnerabilityItems(vulnerabilityCount); + var packageCatalogItem = new PackageCatalogItem( + nupkgMetadata, + test.FeedPackageDetails.CreatedDate, + test.FeedPackageDetails.LastEditedDate, + test.FeedPackageDetails.PublishedDate, + deprecation: deprecationItem, + vulnerabilities: vulnerabilities); + + test.PackageCatalogItemCreator.Setup(x => x.CreateAsync( + It.Is(details => details == test.FeedPackageDetails), + It.Is(timestamp => timestamp == test.UtcNow), + It.IsAny())) + .ReturnsAsync(packageCatalogItem); + + test.Storage.SetupGet(x => x.BaseAddress) + .Returns(test.CatalogBaseUri); + + var blobs = new List(); + + test.Storage.Setup(x => x.SaveAsync( + It.IsNotNull(), + It.IsNotNull(), + It.IsAny())) + .Callback((uri, content, token) => + { + blobs.Add(new CatalogBlob(uri, content)); + }) + .Returns(Task.FromResult(0)); + + test.Storage.Setup(x => x.LoadStringAsync( + It.Is(uri => uri == test.CatalogIndexUri), + It.IsAny())) + .ReturnsAsync(CatalogTestData.GetBeforeIndex(test.CatalogIndexUri).ToString()); + + test.TelemetryService.Setup(x => x.TrackCatalogIndexWriteDuration( + It.Is(duration => duration > TimeSpan.Zero), + It.Is(uri => uri == test.CatalogIndexUri))); + + // Act + var result = await test.WritePackageDetailsToCatalogAsync(); + + // Assert + Assert.Equal(test.UtcNow, result); + + Assert.Equal(3, blobs.Count); + + var catalogPackageDetailsUri = new Uri($"{test.CatalogBaseUri}data/{packageCatalogItem.TimeStamp.ToString("yyyy.MM.dd.HH.mm.ss")}/newtonsoft.json.9.0.2-beta1.json"); + var catalogPageUri = new Uri($"{test.CatalogBaseUri}page0.json"); + + // Verify package details blob + Assert.Equal(catalogPackageDetailsUri, blobs[0].Uri); + Assert.IsType(blobs[0].Content); + + var stringContent = (StringStorageContent)blobs[0].Content; + + Assert.Equal("no-store", stringContent.CacheControl); + Assert.Equal("application/json", stringContent.ContentType); + + var expectedContent = CatalogTestData.GetPackageDetails( + catalogPackageDetailsUri, + packageCatalogItem.CommitId, + packageCatalogItem.TimeStamp, + packageCatalogItem.CreatedDate.Value, + packageCatalogItem.LastEditedDate.Value, + packageCatalogItem.PublishedDate.Value, + deprecationItem, + vulnerabilities); + + var actualContent = JObject.Parse(stringContent.Content); + + Assert.Equal(expectedContent.ToString(), actualContent.ToString()); + + // Verify page blob + Assert.Equal(catalogPageUri, blobs[1].Uri); + Assert.IsType(blobs[1].Content); + + var jtokenContent = (JTokenStorageContent)blobs[1].Content; + + expectedContent = CatalogTestData.GetPage( + catalogPageUri, + packageCatalogItem.CommitId, + packageCatalogItem.TimeStamp, + test.CatalogIndexUri, + catalogPackageDetailsUri); + + Assert.Equal("no-store", jtokenContent.CacheControl); + Assert.Equal("application/json", jtokenContent.ContentType); + Assert.Equal(expectedContent.ToString(), jtokenContent.Content.ToString()); + + // Verify index blob + Assert.Equal(test.CatalogIndexUri, blobs[2].Uri); + Assert.IsType(blobs[2].Content); + + jtokenContent = (JTokenStorageContent)blobs[2].Content; + + var lastEdited = createdPackages ? test.LastEdited : test.UtcNow; + var lastCreated = updateCreatedFromEdited ? lastEdited : (createdPackages ? test.UtcNow : test.LastCreated); + + expectedContent = CatalogTestData.GetAfterIndex( + test.CatalogIndexUri, + packageCatalogItem.CommitId, + packageCatalogItem.TimeStamp, + lastCreated, + test.LastDeleted, + lastEdited, + catalogPageUri); + + Assert.Equal("no-store", jtokenContent.CacheControl); + Assert.Equal("application/json", jtokenContent.ContentType); + Assert.Equal(expectedContent.ToString(), jtokenContent.Content.ToString()); + } + } + } + + private sealed class WritePackageDetailsToCatalogAsyncTest : IDisposable + { + private bool _isDisposed; + + internal DateTime UtcNow { get; } + internal DateTime LastCreated { get; } + internal DateTime LastEdited { get; } + internal DateTime LastDeleted { get; } + internal bool? CreatedPackages { get; set; } + internal bool UpdateCreatedFromEdited { get; set; } + internal Mock PackageCatalogItemCreator { get; } + internal SortedList> Packages { get; } + internal Mock Storage { get; } + internal Mock TelemetryService { get; } + internal Mock Logger { get; } + internal FeedPackageDetails FeedPackageDetails { get; } + internal Uri CatalogBaseUri { get; } + internal Uri CatalogIndexUri { get; } + + internal WritePackageDetailsToCatalogAsyncTest() + { + UtcNow = DateTime.UtcNow; + LastCreated = UtcNow.AddHours(-3); + LastEdited = UtcNow.AddHours(-2); + LastDeleted = UtcNow.AddHours(-1); + + Packages = new SortedList>(); + + PackageCatalogItemCreator = new Mock(MockBehavior.Strict); + Storage = new Mock(MockBehavior.Strict); + TelemetryService = new Mock(MockBehavior.Strict); + Logger = new Mock(); + + CatalogBaseUri = new Uri("https://nuget.test/v3-catalog0/"); + CatalogIndexUri = new Uri(CatalogBaseUri, "index.json"); + + Storage.Setup(x => x.ResolveUri( + It.Is(relativeUri => relativeUri == "index.json"))) + .Returns(CatalogIndexUri); + + FeedPackageDetails = new FeedPackageDetails( + new Uri("https://nuget.test/packages/a"), + UtcNow.AddMinutes(-45), + UtcNow.AddMinutes(-30), + UtcNow.AddMinutes(-15), + packageId: "a", + packageNormalizedVersion: "1.0.0", + packageFullVersion: "1.0.0.0"); + } + + public void Dispose() + { + if (!_isDisposed) + { + PackageCatalogItemCreator.VerifyAll(); + Storage.VerifyAll(); + TelemetryService.VerifyAll(); + Logger.VerifyAll(); + + GC.SuppressFinalize(this); + + _isDisposed = true; + } + } + + internal Task WritePackageDetailsToCatalogAsync() + { + const int maxDegreeOfParallelism = 1; + + return CatalogWriterHelper.WritePackageDetailsToCatalogAsync( + PackageCatalogItemCreator.Object, + Packages, + Storage.Object, + LastCreated, + LastEdited, + LastDeleted, + maxDegreeOfParallelism, + CreatedPackages, + UpdateCreatedFromEdited, + CancellationToken.None, + TelemetryService.Object, + Logger.Object); + } + } + + private sealed class CatalogBlob + { + internal Uri Uri { get; } + internal StorageContent Content { get; } + + internal CatalogBlob(Uri uri, StorageContent content) + { + Uri = uri; + Content = content; + } + } + + public enum PackageDeprecationItemState + { + NotDeprecated, + DeprecatedWithSingleReason, + DeprecatedWithMessage, + DeprecatedWithAlternate + } + + private static PackageDeprecationItem GetPackageDeprecationItemFromState(PackageDeprecationItemState deprecationState) + { + if (deprecationState == PackageDeprecationItemState.NotDeprecated) + { + return null; + } + + var reasons = new[] { "first", "second" }; + string message = null; + string altId = null; + string altVersion = null; + if (deprecationState == PackageDeprecationItemState.DeprecatedWithSingleReason) + { + reasons = reasons.Take(1).ToArray(); + } + + if (deprecationState == PackageDeprecationItemState.DeprecatedWithMessage) + { + message = "this is the message"; + } + + if (deprecationState == PackageDeprecationItemState.DeprecatedWithAlternate) + { + altId = "theId"; + altVersion = "[2.4.5, 2.4.5]"; + } + + return new PackageDeprecationItem(reasons, message, altId, altVersion); + } + + private static IList CreateTestPackageVulnerabilityItems(int vulnerabilityCount) + { + if (vulnerabilityCount == 0) + { + return null; + } + + var vulnerabilities = new List(); + for (int i = 0; i < vulnerabilityCount; i++) + { + vulnerabilities.Add(new PackageVulnerabilityItem("" + 100 + i, "http://www.foo.com/advisory" + i + ".html", "" + i)); + } + + return vulnerabilities; + } + + private static NupkgMetadata GetNupkgMetadata(string resourceName) + { + using (var stream = TestHelper.GetStream(resourceName)) + { + return Utils.GetNupkgMetadata(stream, packageHash: null); + } + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/Db2CatalogCursorTests.cs b/tests/CatalogTests/Helpers/Db2CatalogCursorTests.cs new file mode 100644 index 000000000..626c4ddb7 --- /dev/null +++ b/tests/CatalogTests/Helpers/Db2CatalogCursorTests.cs @@ -0,0 +1,49 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Data.SqlTypes; +using System.Linq; +using NuGet.Services.Metadata.Catalog.Helpers; +using Xunit; + +namespace CatalogTests.Helpers +{ + public class Db2CatalogCursorTests + { + [Theory] + [MemberData(nameof(CursorMethodToColumnNameMappings))] + public void TargetsCreatedColumn(Func cursorMethod, string expectedColumnName) + { + const int top = 1; + var since = DateTime.UtcNow; + var actual = cursorMethod(since, top); + + Assert.Equal(expectedColumnName, actual.ColumnName); + Assert.Equal(since, actual.CursorValue); + Assert.Equal(top, actual.Top); + } + + [Theory] + [MemberData(nameof(CursorMethods))] + public void ProtectsAgainstSqlMinDate(Func cursorMethod) + { + const int top = 1; + var since = new DateTime(SqlDateTime.MinValue.Value.Ticks - 1, DateTimeKind.Utc); + var actual = cursorMethod(since, top); + + Assert.Equal(SqlDateTime.MinValue.Value, actual.CursorValue); + } + + public static IEnumerable CursorMethodToColumnNameMappings => new[] + { + new object[] { (Func)((since, top) => Db2CatalogCursor.ByCreated(since, top)), Db2CatalogProjectionColumnNames.Created }, + new object[] { (Func)((since, top) => Db2CatalogCursor.ByLastEdited(since, top)), Db2CatalogProjectionColumnNames.LastEdited } + }; + + public static IEnumerable CursorMethods => + from testData in CursorMethodToColumnNameMappings + select new object[] { testData[0] }; + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/Db2CatalogProjectionTests.cs b/tests/CatalogTests/Helpers/Db2CatalogProjectionTests.cs new file mode 100644 index 000000000..478130cf9 --- /dev/null +++ b/tests/CatalogTests/Helpers/Db2CatalogProjectionTests.cs @@ -0,0 +1,309 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Data; +using System.Data.Common; +using System.Linq; +using Moq; +using NuGet.Services.Entities; +using NuGet.Services.Metadata.Catalog.Helpers; +using Xunit; +using Constants = NuGet.Services.Metadata.Catalog.Constants; + +namespace CatalogTests.Helpers +{ + public class Db2CatalogProjectionTests + { + [Fact] + public void UnpublishedDateDidNotChange() + { + var expected = new DateTime(1900, 1, 1, 0, 0, 0); + Assert.Equal(expected, Constants.UnpublishedDate); + } + + public class TheConstructor + { + [Fact] + public void ThrowsForNullArgument() + { + Assert.Throws(() => new Db2CatalogProjection(null)); + } + } + + public class TheReadFeedPackageDetailsFromDataReaderMethod + { + private const string PackageContentUrlFormat = "https://unittest.org/packages/{id-lower}/{version-lower}.nupkg"; + private readonly PackageContentUriBuilder _packageContentUriBuilder; + private readonly Db2CatalogProjection _db2catalogProjection; + + public TheReadFeedPackageDetailsFromDataReaderMethod() + { + _packageContentUriBuilder = new PackageContentUriBuilder(PackageContentUrlFormat); + _db2catalogProjection = new Db2CatalogProjection(_packageContentUriBuilder); + } + + [Fact] + public void ThrowsForNullArgument() + { + Assert.Throws(() => _db2catalogProjection.ReadFeedPackageDetailsFromDataReader(null)); + } + + [Theory] + [InlineData(true, true)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(false, false)] + public void PerformsCorrectProjections(bool listed, bool hideLicenseReport) + { + // Arrange + const string packageId = "Package.Id"; + const string normalizedPackageVersion = "1.0.0"; + const string fullPackageVersion = "1.0.0.0"; + const string licenseNames = "MIT"; + const string licenseReportUrl = "https://unittest.org/licenses/MIT"; + const bool requiresLicenseAcceptance = true; + + var utcNow = DateTime.UtcNow; + var createdDate = utcNow.AddDays(-1); + var lastEditedDate = utcNow.AddMinutes(-5); + var publishedDate = createdDate; + var expectedContentUri = _packageContentUriBuilder.Build(packageId, normalizedPackageVersion); + var expectedPublishedDate = listed ? publishedDate : Constants.UnpublishedDate; + var expectedLicenseNames = hideLicenseReport ? null : licenseNames; + var expectedLicenseReportUrl = hideLicenseReport ? null : licenseReportUrl; + var expectedRequiresLicenseAcceptance = true; + + var dataRecordMock = MockDataReader( + packageId, + normalizedPackageVersion, + fullPackageVersion, + createdDate, + lastEditedDate, + publishedDate, + listed, + hideLicenseReport, + licenseNames, + licenseReportUrl, + requiresLicenseAcceptance); + + // Act + var projection = _db2catalogProjection.ReadFeedPackageDetailsFromDataReader(dataRecordMock.Object); + + // Assert + Assert.Equal(packageId, projection.PackageId); + Assert.Equal(normalizedPackageVersion, projection.PackageNormalizedVersion); + Assert.Equal(fullPackageVersion, projection.PackageFullVersion); + Assert.Equal(createdDate, projection.CreatedDate); + Assert.Equal(lastEditedDate, projection.LastEditedDate); + Assert.Equal(expectedPublishedDate, projection.PublishedDate); + Assert.Equal(expectedContentUri, projection.ContentUri); + Assert.Equal(expectedLicenseNames, projection.LicenseNames); + Assert.Equal(expectedLicenseReportUrl, projection.LicenseReportUrl); + Assert.Equal(expectedRequiresLicenseAcceptance, projection.RequiresLicenseAcceptance); + Assert.Null(projection.DeprecationInfo); + } + + private static Mock MockDataReader( + string packageId, + string normalizedPackageVersion, + string fullPackageVersion, + DateTime createdDate, + DateTime lastEditedDate, + DateTime publishedDate, + bool listed, + bool hideLicenseReport, + string licenseNames, + string licenseReportUrl, + bool requiresLicenseAcceptance) + { + const int ordinalCreated = 3; + const int ordinalLastEdited = 4; + const int ordinalPublished = 5; + const int ordinalListed = 6; + const int ordinalHideLicenseReport = 7; + const int ordinalRequiresLicenseAcceptance = 10; + + var dataReaderMock = new Mock(MockBehavior.Strict); + + dataReaderMock.SetupGet(m => m[Db2CatalogProjectionColumnNames.PackageId]).Returns(packageId); + dataReaderMock.SetupGet(m => m[Db2CatalogProjectionColumnNames.NormalizedVersion]).Returns(normalizedPackageVersion); + dataReaderMock.SetupGet(m => m[Db2CatalogProjectionColumnNames.FullVersion]).Returns(fullPackageVersion); + + dataReaderMock.SetupGet(m => m[Db2CatalogProjectionColumnNames.LicenseNames]).Returns(licenseNames); + dataReaderMock.SetupGet(m => m[Db2CatalogProjectionColumnNames.LicenseReportUrl]).Returns(licenseReportUrl); + + dataReaderMock.Setup(m => m.GetOrdinal(Db2CatalogProjectionColumnNames.Listed)).Returns(ordinalListed); + dataReaderMock.Setup(m => m.GetBoolean(ordinalListed)).Returns(listed); + + dataReaderMock.Setup(m => m.GetOrdinal(Db2CatalogProjectionColumnNames.HideLicenseReport)).Returns(ordinalHideLicenseReport); + dataReaderMock.Setup(m => m.GetBoolean(ordinalHideLicenseReport)).Returns(hideLicenseReport); + + dataReaderMock.SetupGet(m => m[Db2CatalogProjectionColumnNames.Created]).Returns(createdDate); + dataReaderMock.Setup(m => m.GetOrdinal(Db2CatalogProjectionColumnNames.Created)).Returns(ordinalCreated); + dataReaderMock.Setup(m => m.GetDateTime(ordinalCreated)).Returns(createdDate); + + dataReaderMock.SetupGet(m => m[Db2CatalogProjectionColumnNames.LastEdited]).Returns(lastEditedDate); + dataReaderMock.Setup(m => m.GetOrdinal(Db2CatalogProjectionColumnNames.LastEdited)).Returns(ordinalLastEdited); + dataReaderMock.Setup(m => m.GetDateTime(ordinalLastEdited)).Returns(lastEditedDate); + + dataReaderMock.SetupGet(m => m[Db2CatalogProjectionColumnNames.Published]).Returns(publishedDate); + dataReaderMock.Setup(m => m.GetOrdinal(Db2CatalogProjectionColumnNames.Published)).Returns(ordinalPublished); + dataReaderMock.Setup(m => m.GetDateTime(ordinalPublished)).Returns(publishedDate); + + dataReaderMock.SetupGet(m => m[Db2CatalogProjectionColumnNames.RequiresLicenseAcceptance]).Returns(requiresLicenseAcceptance); + dataReaderMock.Setup(m => m.GetOrdinal(Db2CatalogProjectionColumnNames.RequiresLicenseAcceptance)).Returns(ordinalRequiresLicenseAcceptance); + dataReaderMock.Setup(m => m.GetBoolean(ordinalRequiresLicenseAcceptance)).Returns(requiresLicenseAcceptance); + + // Simulate that these columns do not exist in the resultset. + dataReaderMock.Setup(m => m.GetOrdinal(Db2CatalogProjectionColumnNames.DeprecationStatus)).Throws(); + + return dataReaderMock; + } + } + + public class TheReadDeprecationInfoFromDataReaderMethod + { + private const string PackageContentUrlFormat = "https://unittest.org/packages/{id-lower}/{version-lower}.nupkg"; + private readonly PackageContentUriBuilder _packageContentUriBuilder; + private readonly Db2CatalogProjection _db2catalogProjection; + + public TheReadDeprecationInfoFromDataReaderMethod() + { + _packageContentUriBuilder = new PackageContentUriBuilder(PackageContentUrlFormat); + _db2catalogProjection = new Db2CatalogProjection(_packageContentUriBuilder); + } + + [Fact] + public void ThrowsForNullArgument() + { + Assert.Throws(() => _db2catalogProjection.ReadDeprecationInfoFromDataReader(null)); + } + + [Theory] + [InlineData(null, null)] + [InlineData(null, "1.0.0")] + [InlineData("alternate.package.id", null)] + [InlineData("alternate.package.id", "1.0.0")] + public void PerformsCorrectProjections(string alternatePackageId, string alternatePackageVersionRange) + { + foreach (var packageDeprecationStatus in Enum.GetValues(typeof(PackageDeprecationStatus)).Cast()) + { + VerifyDeprecationProjections( + packageDeprecationStatus, + alternatePackageId, + alternatePackageVersionRange); + } + } + + private void VerifyDeprecationProjections( + PackageDeprecationStatus status, + string alternatePackageId, + string alternatePackageVersionRange) + { + // Arrange + string customMessage = null; + + Mock dataReaderMock; + if (status == PackageDeprecationStatus.NotDeprecated) + { + dataReaderMock = MockDataReader(status); + } + else + { + customMessage = "custom message"; + + dataReaderMock = MockDataReader( + status, + customMessage, + alternatePackageId, + alternatePackageVersionRange); + } + + // Act + var projection = _db2catalogProjection.ReadDeprecationInfoFromDataReader(dataReaderMock.Object); + + // Assert + if (status == PackageDeprecationStatus.NotDeprecated) + { + Assert.Null(projection); + } + else + { + Assert.NotNull(projection); + + foreach (var deprecationStatusFlag in Enum.GetValues(typeof(PackageDeprecationStatus)) + .Cast() + .Except(new[] { PackageDeprecationStatus.NotDeprecated })) + { + if (status.HasFlag(deprecationStatusFlag)) + { + Assert.Contains(deprecationStatusFlag.ToString(), projection.Reasons); + } + else + { + Assert.DoesNotContain(deprecationStatusFlag.ToString(), projection.Reasons); + } + } + + Assert.Equal(customMessage, projection.Message); + Assert.Equal(alternatePackageId, projection.AlternatePackageId); + + if (alternatePackageId != null && alternatePackageVersionRange == null) + { + Assert.Equal(Db2CatalogProjection.AlternatePackageVersionWildCard, projection.AlternatePackageRange); + } + else if (alternatePackageId == null) + { + Assert.Null(projection.AlternatePackageRange); + } + else + { + Assert.Equal( + $"[{alternatePackageVersionRange}, )", + projection.AlternatePackageRange); + } + } + } + + private static Mock MockDataReader( + PackageDeprecationStatus deprecationStatus, + string message = null, + string alternatePackageId = null, + string alternatePackageVersionRange = null) + { + var dataReaderMock = new Mock(MockBehavior.Strict); + + if (deprecationStatus == PackageDeprecationStatus.NotDeprecated) + { + // Simulate that these columns do not exist in the resultset. + dataReaderMock.Setup(m => m.GetOrdinal(Db2CatalogProjectionColumnNames.DeprecationStatus)).Throws(); + } + else + { + const int ordinalDeprecationStatus = 7; + dataReaderMock.Setup(m => m.GetOrdinal(Db2CatalogProjectionColumnNames.DeprecationStatus)).Returns(ordinalDeprecationStatus); + dataReaderMock.Setup(m => m.IsDBNull(ordinalDeprecationStatus)).Returns(false); + dataReaderMock.Setup(m => m.GetInt32(ordinalDeprecationStatus)).Returns((int)deprecationStatus); + + const int ordinalDeprecationMessage = 8; + dataReaderMock.Setup(m => m.GetOrdinal(Db2CatalogProjectionColumnNames.DeprecationMessage)).Returns(ordinalDeprecationMessage); + dataReaderMock.Setup(m => m.IsDBNull(ordinalDeprecationMessage)).Returns(message == null); + dataReaderMock.Setup(m => m.GetString(ordinalDeprecationMessage)).Returns(message); + + const int ordinalAlternatePackageId = 9; + dataReaderMock.Setup(m => m.GetOrdinal(Db2CatalogProjectionColumnNames.AlternatePackageId)).Returns(ordinalAlternatePackageId); + dataReaderMock.Setup(m => m.IsDBNull(ordinalAlternatePackageId)).Returns(alternatePackageId == null); + dataReaderMock.Setup(m => m.GetString(ordinalAlternatePackageId)).Returns(alternatePackageId); + + const int ordinalAlternatePackageVersionRange = 10; + dataReaderMock.Setup(m => m.GetOrdinal(Db2CatalogProjectionColumnNames.AlternatePackageVersion)).Returns(ordinalAlternatePackageVersionRange); + dataReaderMock.Setup(m => m.IsDBNull(ordinalAlternatePackageVersionRange)).Returns(alternatePackageVersionRange == null); + dataReaderMock.Setup(m => m.GetString(ordinalAlternatePackageVersionRange)).Returns(alternatePackageVersionRange); + } + + return dataReaderMock; + } + } + } +} diff --git a/tests/CatalogTests/Helpers/FeedHelpersTests.cs b/tests/CatalogTests/Helpers/FeedHelpersTests.cs new file mode 100644 index 000000000..8867e34d0 --- /dev/null +++ b/tests/CatalogTests/Helpers/FeedHelpersTests.cs @@ -0,0 +1,232 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using NgTests.Infrastructure; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Versioning; +using Xunit; + +namespace CatalogTests.Helpers +{ + public class FeedHelpersTests + { + private const string _baseUri = "http://unit.test"; + + public static IEnumerable GetPackages_GetsAllPackages_data + { + get + { + yield return new object[] + { + ODataPackages + }; + } + } + + [Theory] + [MemberData(nameof(GetPackages_GetsAllPackages_data))] + public async Task GetPackages_GetsAllPackages(IEnumerable oDataPackages) + { + // Act + var feedPackages = await TestGetPackagesAsync(oDataPackages); + + // Assert + foreach (var oDataPackage in oDataPackages) + { + Assert.Contains(feedPackages, + (feedPackage) => + { + return ArePackagesEqual(feedPackage, oDataPackage); + }); + } + + foreach (var feedPackage in feedPackages) + { + VerifyDateTimesAreInUtc(feedPackage); + } + } + + private Task> TestGetPackagesAsync(IEnumerable oDataPackages) + { + return FeedHelpers.GetPackages( + new HttpClient(GetMessageHandlerForPackages(oDataPackages)), + new Uri(_baseUri + "/test")); + } + + private HttpMessageHandler GetMessageHandlerForPackages(IEnumerable oDataPackages) + { + var mockServer = new MockServerHttpClientHandler(); + + mockServer.SetAction("/test", (request) => + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + ODataFeedHelper.ToODataFeed(oDataPackages, new Uri(_baseUri), "Packages")) + }); + }); + + return mockServer; + } + + private bool ArePackagesEqual(FeedPackageDetails feedPackage, ODataPackage oDataPackage) + { + return + feedPackage.PackageId == oDataPackage.Id && + feedPackage.PackageNormalizedVersion == oDataPackage.Version && + feedPackage.ContentUri.ToString() == $"{_baseUri}/package/{oDataPackage.Id}/{NuGetVersion.Parse(oDataPackage.Version).ToNormalizedString()}" && + feedPackage.CreatedDate.Ticks == oDataPackage.Created.Ticks && + feedPackage.LastEditedDate.Ticks == oDataPackage.LastEdited.Value.Ticks && + feedPackage.PublishedDate.Ticks == oDataPackage.Published.Ticks && + feedPackage.LicenseNames == oDataPackage.LicenseNames && + feedPackage.LicenseReportUrl == oDataPackage.LicenseReportUrl; + } + + private static IEnumerable ODataPackages + { + get + { + return new List + { + new ODataPackage + { + Id = "listedPackage", + Version = "1.0.0", + Listed = true, + + Created = new DateTime(2017, 4, 6, 15, 10, 0), + LastEdited = new DateTime(2017, 4, 6, 15, 10, 1), + Published = new DateTime(2017, 4, 6, 15, 10, 0), + + LicenseNames = "ABCD", + LicenseReportUrl = "https://unit.test/license" + }, + + new ODataPackage + { + Id = "unlistedPackage", + Version = "2.0.1", + Listed = false, + + Created = new DateTime(2017, 4, 6, 15, 12, 0), + LastEdited = new DateTime(2017, 4, 6, 15, 13, 1), + Published = Convert.ToDateTime("1900-01-01T00:00:00Z"), + + LicenseNames = "ABCD", + LicenseReportUrl = "https://unit.test/license" + }, + + new ODataPackage + { + Id = "listedPackage", + Version = "2.1.1", + Listed = true, + + Created = new DateTime(2017, 4, 6, 15, 13, 3), + LastEdited = new DateTime(2017, 4, 6, 15, 14, 1), + Published = new DateTime(2017, 4, 6, 15, 13, 0), + + LicenseNames = "ABCD", + LicenseReportUrl = "https://unit.test/license" + }, + + new ODataPackage + { + Id = "unlistedPackage", + Version = "3.0.4", + Listed = false, + + Created = new DateTime(2017, 4, 6, 15, 15, 3), + LastEdited = new DateTime(2017, 4, 6, 15, 16, 4), + Published = Convert.ToDateTime("1900-01-01T00:00:00Z"), + + LicenseNames = "ABCD", + LicenseReportUrl = "https://unit.test/license" + }, + + new ODataPackage + { + Id = "packageWithPrerelease", + Version = "2.2.2-abcdef", + Listed = true, + + Created = new DateTime(2017, 4, 6, 16, 30, 0), + LastEdited = new DateTime(2017, 4, 6, 17, 45, 1), + Published = new DateTime(2017, 4, 6, 16, 30, 0), + + LicenseNames = "ABCD", + LicenseReportUrl = "https://unit.test/license" + }, + + new ODataPackage + { + Id = "packageWithNormalized", + Version = "4.4.4.0", + Listed = true, + + Created = new DateTime(2017, 4, 6, 20, 13, 25), + LastEdited = new DateTime(2017, 4, 6, 20, 37, 47), + Published = new DateTime(2017, 4, 6, 20, 13, 25), + + LicenseNames = "ABCD", + LicenseReportUrl = "https://unit.test/license" + }, + + new ODataPackage + { + Id = "packageWithNormalizedPrerelease", + Version = "5.6.3.0-laskdfj224", + Listed = true, + + Created = new DateTime(2017, 4, 7, 3, 13, 25), + LastEdited = new DateTime(2017, 4, 7, 4, 52, 55), + Published = new DateTime(2017, 4, 6, 3, 13, 25), + + LicenseNames = "ABCD", + LicenseReportUrl = "https://unit.test/license" + }, + + new ODataPackage + { + Id = "listedPackageWithDuplicate", + Version = "1.0.1", + Listed = true, + + Created = new DateTime(2017, 5, 1, 0, 0, 0), + LastEdited = new DateTime(2017, 5, 1, 0, 0, 0), + Published = new DateTime(2017, 5, 1, 0, 0, 0), + + LicenseNames = "ABCD", + LicenseReportUrl = "https://unit.test/license" + }, + + new ODataPackage + { + Id = "listedPackageWithDuplicate", + Version = "1.0.2", + Listed = true, + + Created = new DateTime(2017, 5, 1, 0, 0, 0), + LastEdited = new DateTime(2017, 5, 1, 0, 0, 0), + Published = new DateTime(2017, 5, 1, 0, 0, 0), + + LicenseNames = "ABCD", + LicenseReportUrl = "https://unit.test/license" + } + }; + } + } + + private static void VerifyDateTimesAreInUtc(FeedPackageDetails feedPackage) + { + Assert.Equal(DateTimeKind.Utc, feedPackage.CreatedDate.Kind); + Assert.Equal(DateTimeKind.Utc, feedPackage.LastEditedDate.Kind); + Assert.Equal(DateTimeKind.Utc, feedPackage.PublishedDate.Kind); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/FeedPackageIdentityTests.cs b/tests/CatalogTests/Helpers/FeedPackageIdentityTests.cs new file mode 100644 index 000000000..7b2d17bec --- /dev/null +++ b/tests/CatalogTests/Helpers/FeedPackageIdentityTests.cs @@ -0,0 +1,90 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Versioning; +using Xunit; + +namespace CatalogTests.Helpers +{ + public class FeedPackageIdentityTests + { + [Flags] + private enum Equals_Data_States + { + IdIsWrong = 1, + IdIsDifferentCase = 2, + VersionIsWrong = 4, + VersionIsDifferentCase = 8 + } + + public static IEnumerable Equals_Data + { + get + { + const string idA = "a"; + const string versionA = "1.0.0-ab"; + + for (int i = 0; i < (int)Enum.GetValues(typeof(Equals_Data_States)).Cast().Max() * 2; i++) + { + var idB = (i & (int)Equals_Data_States.IdIsWrong) == 0 ? "a" : "b"; + + if ((i & (int)Equals_Data_States.IdIsDifferentCase) != 0) + { + idB = idB.ToUpperInvariant(); + } + + var versionB = (i & (int)Equals_Data_States.VersionIsWrong) == 0 ? "1.0.0-ab" : "1.0.1-ab"; + + if ((i & (int)Equals_Data_States.VersionIsDifferentCase) != 0) + { + versionB = versionB.ToUpperInvariant(); + } + + var success = (i & (int)Equals_Data_States.IdIsWrong) == 0 && (i & (int)Equals_Data_States.VersionIsWrong) == 0; + + yield return new object[] { idA, versionA, idB, versionB, success }; + } + } + } + + [Theory] + [MemberData(nameof(Equals_Data))] + public void FeedPackageIdentityEquals(string idA, string versionA, string idB, string versionB, bool success) + { + var packageA = new FeedPackageIdentity(idA, versionA); + var packageB = new FeedPackageIdentity(idB, versionB); + + Assert.Equal(success, packageA.Equals(packageB)); + Assert.Equal(success, packageB.Equals(packageA)); + + Assert.Equal(success, packageA.GetHashCode() == packageB.GetHashCode()); + } + + [Fact] + public void PackageIdentityConstructorUsesFullString() + { + const string id = "id"; + const string versionString = "1.0.0+buildmetadata"; + var package = new PackageIdentity(id, NuGetVersion.Parse(versionString)); + + var feedPackage = new FeedPackageIdentity(package); + var equivalentFeedPackage1 = new FeedPackageIdentity(id, versionString); + var equivalentFeedPackage2 = new FeedPackageIdentity(package.Id, package.Version.ToFullString()); + var differentFeedPackage = new FeedPackageIdentity(package.Id, package.Version.ToNormalizedString()); + + foreach (var equivalentFeedPackage in new FeedPackageIdentity[] { equivalentFeedPackage1, equivalentFeedPackage2 }) + { + Assert.True(feedPackage.Equals(equivalentFeedPackage)); + Assert.Equal(feedPackage.GetHashCode(), equivalentFeedPackage.GetHashCode()); + } + + Assert.False(feedPackage.Equals(differentFeedPackage)); + Assert.NotEqual(feedPackage.GetHashCode(), differentFeedPackage.GetHashCode()); + } + } +} diff --git a/tests/CatalogTests/Helpers/GalleryDatabaseQueryServiceTests.cs b/tests/CatalogTests/Helpers/GalleryDatabaseQueryServiceTests.cs new file mode 100644 index 000000000..58ee61fcf --- /dev/null +++ b/tests/CatalogTests/Helpers/GalleryDatabaseQueryServiceTests.cs @@ -0,0 +1,303 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using NuGet.Services.Entities; +using NuGet.Services.Metadata.Catalog.Helpers; +using Xunit; +using CatalogConstants = NuGet.Services.Metadata.Catalog.Constants; + +namespace CatalogTests.Helpers +{ + public class GalleryDatabaseQueryServiceTests + { + public class TheBuildDB2CatalogSqlQueryMethod + { + [Fact] + public void SelectsTopWithTies() + { + // Arrange + var cursor = Db2CatalogCursor.ByCreated(DateTime.UtcNow, 10); + + // Act + var queryString = GalleryDatabaseQueryService.BuildDb2CatalogSqlQuery(cursor); + + // Assert + Assert.Contains($"SELECT TOP {cursor.Top} WITH TIES", queryString); + } + + [Fact] + public void OrdersByCursorColumnName_Created() + { + // Arrange + var cursor = Db2CatalogCursor.ByCreated(DateTime.UtcNow, 20); + + // Act + var queryString = GalleryDatabaseQueryService.BuildDb2CatalogSqlQuery(cursor); + + // Assert + Assert.EndsWith($"ORDER BY P_EXT.[{cursor.ColumnName}], P_EXT.[{Db2CatalogProjectionColumnNames.Key}]", queryString); + } + + [Fact] + public void OrdersByCursorColumnName_LastEdited() + { + // Arrange + var cursor = Db2CatalogCursor.ByLastEdited(DateTime.UtcNow, 20); + + // Act + var queryString = GalleryDatabaseQueryService.BuildDb2CatalogSqlQuery(cursor); + + // Assert + Assert.EndsWith($"ORDER BY P_EXT.[{cursor.ColumnName}], P_EXT.[{Db2CatalogProjectionColumnNames.Key}]", queryString); + } + + [Fact] + public void SelectsFromPackagesTable() + { + // Arrange + var cursor = Db2CatalogCursor.ByCreated(DateTime.UtcNow, 20); + + // Act + var queryString = GalleryDatabaseQueryService.BuildDb2CatalogSqlQuery(cursor); + + // Assert + Assert.Contains("FROM [dbo].[Packages] AS P", queryString); + } + + [Fact] + public void JoinsWithPackageRegistrationsTable() + { + // Arrange + var cursor = Db2CatalogCursor.ByCreated(DateTime.UtcNow, 20); + + // Act + var queryString = GalleryDatabaseQueryService.BuildDb2CatalogSqlQuery(cursor); + + // Assert + Assert.Contains("INNER JOIN [dbo].[PackageRegistrations] AS PR ON P.[PackageRegistrationKey] = PR.[Key]", queryString); + } + + [Fact] + public void JoinsWithPackageVulnerabilitiesTables() + { + // Arrange + var cursor = Db2CatalogCursor.ByCreated(DateTime.UtcNow, 20); + + // Act + var queryString = GalleryDatabaseQueryService.BuildDb2CatalogSqlQuery(cursor); + + // Assert + Assert.Contains("LEFT JOIN [dbo].[VulnerablePackageVersionRangePackages] AS VPVRP ON VPVRP.[Package_Key] = P_EXT.[Key]", queryString); + Assert.Contains("LEFT JOIN [dbo].[VulnerablePackageVersionRanges] AS VPVR ON VPVR.[Key] = VPVRP.[VulnerablePackageVersionRange_Key]", queryString); + Assert.Contains("LEFT JOIN [dbo].[PackageVulnerabilities] AS PV ON PV.[Key] = VPVR.[VulnerabilityKey]", queryString); + } + + [Fact] + public void OnlyConsidersPackagesWithAvailableStatus() + { + // Arrange + var cursor = Db2CatalogCursor.ByCreated(DateTime.UtcNow, 20); + + // Act + var queryString = GalleryDatabaseQueryService.BuildDb2CatalogSqlQuery(cursor); + + // Assert + Assert.Contains($"WHERE P.[PackageStatusKey] = {(int)PackageStatus.Available}", queryString); + } + } + + public class TheOrderPackagesByKeyDateMethod + { + [Fact] + public void OrdersBySelectedPropertyDescending() + { + // Arrange + var utcNow = DateTime.UtcNow; + var firstCreatedPackage = new FeedPackageDetails( + new Uri("https://unittest.org/packages/Package.Id/1.0.1"), + createdDate: utcNow.AddDays(-2), + lastEditedDate: utcNow, + publishedDate: utcNow.AddDays(-2), + packageId: "Package.Id", + packageNormalizedVersion: "1.0.1", + packageFullVersion: "1.0.1"); + var firstEditedPackage = new FeedPackageDetails( + new Uri("https://unittest.org/packages/Package.Id/1.0.0"), + createdDate: utcNow.AddDays(-1), + lastEditedDate: utcNow.AddHours(-8), + publishedDate: utcNow.AddDays(-1), + packageId: "Package.Id", + packageNormalizedVersion: "1.0.0", + packageFullVersion: "1.0.0"); + + var packages = new List + { + firstCreatedPackage, + firstEditedPackage + }; + + // Act + var orderedByCreatedDate = GalleryDatabaseQueryService.OrderPackagesByKeyDate(packages, p => p.CreatedDate); + var orderedByLastEditedDate = GalleryDatabaseQueryService.OrderPackagesByKeyDate(packages, p => p.LastEditedDate); + + // Assert + Assert.Equal(firstCreatedPackage.CreatedDate, orderedByCreatedDate.First().Key); + Assert.Equal(1, orderedByCreatedDate.First().Value.Count); + Assert.Contains(firstCreatedPackage, orderedByCreatedDate.First().Value); + + Assert.Equal(firstEditedPackage.LastEditedDate, orderedByLastEditedDate.First().Key); + Assert.Equal(1, orderedByLastEditedDate.First().Value.Count); + Assert.Contains(firstEditedPackage, orderedByLastEditedDate.First().Value); + } + + [Fact] + public void WillSkipLastTimestampIfItWouldResultInPageOverflow() + { + // Arrange + const int top = 20; + var utcNow = DateTime.UtcNow; + var packages = new List(); + + // Ensure the top 19 timestamps have a package each. + for (int i = 1; i < top; i++) + { + packages.Add(new FeedPackageDetails( + new Uri($"https://unittest.org/packages/Package.Id{i}/1.0.{i}"), + createdDate: utcNow.AddDays(-i), + lastEditedDate: utcNow.AddDays(-i), + publishedDate: utcNow.AddDays(-i), + packageId: $"Package.Id{i}", + packageNormalizedVersion: $"1.0.{i}", + packageFullVersion: $"1.0.{i}")); + } + + // Ensure the last timestamp has enough packages to have it exceed the max page size. + // This simulates TOP 20 WITH TIES behavior encountering bulk changes to these packages + var lastTimestamp = utcNow.AddHours(-8); + for (int i = 0; i < CatalogConstants.MaxPageSize - top + 2; i++) + { + packages.Add(new FeedPackageDetails( + new Uri($"https://unittest.org/packages/BatchUpdatedPackage.Id/1.0.{i}"), + createdDate: utcNow.AddHours(-9), + lastEditedDate: lastTimestamp, + publishedDate: utcNow.AddHours(-9), + packageId: "BatchUpdatedPackage.Id", + packageNormalizedVersion: $"1.0.{i}", + packageFullVersion: $"1.0.{i}")); + } + + // Act + var orderedByLastEditedDate = GalleryDatabaseQueryService.OrderPackagesByKeyDate(packages, p => p.LastEditedDate); + + // Assert + Assert.Equal(top - 1, orderedByLastEditedDate.Count); + Assert.DoesNotContain(lastTimestamp, orderedByLastEditedDate.Keys); + Assert.True(orderedByLastEditedDate.Values.Sum(v => v.Count) <= CatalogConstants.MaxPageSize); + } + + [Fact] + public void WillSkipAnyTimestampThatWouldResultInPageOverflow() + { + // Arrange + const int top = 20; + var utcNow = DateTime.UtcNow; + var packages = new List(); + + // Ensure the top 10 timestamps have a package each. + for (int i = 10; i < top; i++) + { + packages.Add(new FeedPackageDetails( + new Uri($"https://unittest.org/packages/Package.Id{i}/1.0.{i}"), + createdDate: utcNow.AddDays(-i), + lastEditedDate: utcNow.AddDays(-i), + publishedDate: utcNow.AddDays(-i), + packageId: $"Package.Id{i}", + packageNormalizedVersion: $"1.0.{i}", + packageFullVersion: $"1.0.{i}")); + } + + // Ensure the next timestamp has enough packages to have it reach the max page size. + // This simulates encountering bulk changes at timestamp 11. + var timestampForBulkChanges = utcNow.AddHours(-8); + for (int i = 0; i < CatalogConstants.MaxPageSize - top + 10; i++) + { + packages.Add(new FeedPackageDetails( + new Uri($"https://unittest.org/packages/BatchUpdatedPackage.Id/1.0.{i}"), + createdDate: utcNow.AddHours(-9), + lastEditedDate: timestampForBulkChanges, + publishedDate: utcNow.AddHours(-9), + packageId: "BatchUpdatedPackage.Id", + packageNormalizedVersion: $"1.0.{i}", + packageFullVersion: $"1.0.{i}")); + } + + // Add some more timestamps with a package each to the end of the top 20 resultset. + for (int i = top - 10 + 1; i < top; i++) + { + packages.Add(new FeedPackageDetails( + new Uri($"https://unittest.org/packages/Another.Package.Id{i}/1.0.{i}"), + createdDate: utcNow.AddDays(-i), + lastEditedDate: utcNow.AddMinutes(-i), + publishedDate: utcNow.AddDays(-i), + packageId: $"Another.Package.Id{i}", + packageNormalizedVersion: $"1.0.{i}", + packageFullVersion: $"1.0.{i}")); + } + + // Act + var orderedByLastEditedDate = GalleryDatabaseQueryService.OrderPackagesByKeyDate(packages, p => p.LastEditedDate); + + // Assert + Assert.Equal(top - 10 + 1, orderedByLastEditedDate.Count); + Assert.Contains(timestampForBulkChanges, orderedByLastEditedDate.Keys); + Assert.True(orderedByLastEditedDate.Values.Sum(v => v.Count) <= CatalogConstants.MaxPageSize); + } + + [Fact] + public void WillNotSkipSingleTimestampIfThatWouldResultInPageOverflow() + { + // Arrange + var utcNow = DateTime.UtcNow; + var packages = new List(); + + // Ensure the first timestamp has enough packages to overflow. + var timestampForBulkChanges = utcNow.AddHours(-8); + for (int i = 0; i < CatalogConstants.MaxPageSize + 1; i++) + { + packages.Add(new FeedPackageDetails( + new Uri($"https://unittest.org/packages/BatchUpdatedPackage.Id{i}/1.0.{i}"), + createdDate: utcNow.AddDays(-i), + lastEditedDate: timestampForBulkChanges, + publishedDate: utcNow.AddDays(-i), + packageId: $"BatchUpdatedPackage.Id{i}", + packageNormalizedVersion: $"1.0.{i}", + packageFullVersion: $"1.0.{i}")); + } + + // Add 19 more timestamps to simulate top 20. + for (int i = 0; i < 19; i++) + { + packages.Add(new FeedPackageDetails( + new Uri($"https://unittest.org/packages/Package.Id/1.0.{i}"), + createdDate: utcNow.AddHours(-9), + lastEditedDate: utcNow.AddMinutes(-i), + publishedDate: utcNow.AddHours(-9), + packageId: "Package.Id", + packageNormalizedVersion: $"1.0.{i}", + packageFullVersion: $"1.0.{i}")); + } + + // Act + var orderedByLastEditedDate = GalleryDatabaseQueryService.OrderPackagesByKeyDate(packages, p => p.LastEditedDate); + + // Assert + Assert.Single(orderedByLastEditedDate); + Assert.Equal(timestampForBulkChanges, orderedByLastEditedDate.Single().Key); + Assert.True(orderedByLastEditedDate.Values.Sum(v => v.Count) >= CatalogConstants.MaxPageSize); + } + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/LicenseHelperTests.cs b/tests/CatalogTests/Helpers/LicenseHelperTests.cs new file mode 100644 index 000000000..6dec5bce4 --- /dev/null +++ b/tests/CatalogTests/Helpers/LicenseHelperTests.cs @@ -0,0 +1,29 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Services.Metadata.Catalog; +using Xunit; + +namespace CatalogTests.Helpers +{ + public class LicenseHelperTests + { + [Theory] + [InlineData("testPackage", "1.0.0", "https://testnuget", "https://testnuget/packages/testPackage/1.0.0/license")] + [InlineData("testPackage", "1.0.0", "https://testnuget/", "https://testnuget/packages/testPackage/1.0.0/license")] + [InlineData("testPackage", "1.0.0", "https://testnuget//", "https://testnuget/packages/testPackage/1.0.0/license")] + [InlineData("testPackage", null, "https://testnuget/", null)] + [InlineData("testPackage", "1.0.0", null, null)] + [InlineData("", "", "https://testnuget/", null)] + [InlineData("测试更新包", "1.0.0", "https://testnuget/", "https://testnuget/packages/%E6%B5%8B%E8%AF%95%E6%9B%B4%E6%96%B0%E5%8C%85/1.0.0/license")] + public void GivenPackageIdAndVersionAndGalleryBaseUrl_ReturnsLicenseUrl(string packageId, string packageVersion, string galleryBaseAddress, string expectedLicenseUrl) + { + // Arrange and Act + var licenseUrl = LicenseHelper.GetGalleryLicenseUrl(packageId, packageVersion, galleryBaseAddress == null ? null : new Uri(galleryBaseAddress)); + + // Assert + Assert.Equal(expectedLicenseUrl, licenseUrl); + } + } +} diff --git a/tests/CatalogTests/Helpers/NuGetVersionUtilityTests.cs b/tests/CatalogTests/Helpers/NuGetVersionUtilityTests.cs new file mode 100644 index 000000000..ba4dcd5be --- /dev/null +++ b/tests/CatalogTests/Helpers/NuGetVersionUtilityTests.cs @@ -0,0 +1,72 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Metadata.Catalog.Helpers; +using Xunit; + +namespace CatalogTests.Helpers +{ + public class NuGetVersionUtilityTests + { + [Theory] + [InlineData("1.0.0-alpha", "1.0.0-alpha")] + [InlineData("1.0.0-alpha.1", "1.0.0-alpha.1")] + [InlineData("1.0.0-alpha+githash", "1.0.0-alpha")] + [InlineData("1.0.0.0", "1.0.0")] + [InlineData("invalid", "invalid")] + public void NormalizeVersion(string input, string expected) + { + // Arrange & Act + var actual = NuGetVersionUtility.NormalizeVersion(input); + + // Assert + Assert.Equal(expected, actual); + } + + [Theory] + [InlineData("[1.0.0-alpha, )", "[1.0.0-alpha, )")] + [InlineData("1.0.0-alpha.1", "[1.0.0-alpha.1, )")] + [InlineData("[1.0.0-alpha+githash, )", "[1.0.0-alpha, )")] + [InlineData("[1.0, 2.0]", "[1.0.0, 2.0.0]")] + [InlineData("invalid", "invalid")] + public void NormalizeVersionRange(string input, string expected) + { + // Arrange + var defaultValue = input; + + // Arrange + var actual = NuGetVersionUtility.NormalizeVersionRange(input, defaultValue); + + // Assert + Assert.Equal(expected, actual); + } + + [Fact] + public void NormalizeVersionRange_UsesDifferentDefault() + { + // Arrange + var input = "invalid"; + var defaultValue = "(, )"; + + // Act + var actual = NuGetVersionUtility.NormalizeVersionRange(input, defaultValue); + + // Assert + Assert.Equal(defaultValue, actual); + } + + [Theory] + [InlineData("1.0.0-alpha.1", "1.0.0-alpha.1")] + [InlineData("1.0.0-alpha+githash", "1.0.0-alpha+githash")] + [InlineData("1.0.0.0", "1.0.0")] + [InlineData("invalid", "invalid")] + public void GetFullVersionString(string input, string expected) + { + // Arrange & Act + var actual = NuGetVersionUtility.GetFullVersionString(input); + + // Assert + Assert.Equal(expected, actual); + } + } +} diff --git a/tests/CatalogTests/Helpers/PackageContentUriBuilderTests.cs b/tests/CatalogTests/Helpers/PackageContentUriBuilderTests.cs new file mode 100644 index 000000000..f31d133f2 --- /dev/null +++ b/tests/CatalogTests/Helpers/PackageContentUriBuilderTests.cs @@ -0,0 +1,55 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Services.Metadata.Catalog.Helpers; +using Xunit; + +namespace CatalogTests.Helpers +{ + public class PackageContentUriBuilderTests + { + [Fact] + public void PlaceholdersDidNotChange() + { + Assert.Equal("{id-lower}", PackageContentUriBuilder.IdLowerPlaceholderString); + Assert.Equal("{version-lower}", PackageContentUriBuilder.VersionLowerPlaceholderString); + } + + public class TheConstructor + { + [Fact] + public void ThrowsForNullArgument() + { + Assert.Throws(() => new PackageContentUriBuilder(null)); + } + } + + public class TheBuildMethod + { + [Theory] + [InlineData(null, "1.0.0")] + [InlineData("Package.Id", null)] + public void ThrowsForNullArguments(string packageId, string normalizedPackageVersion) + { + var packageContentUriBuilder = new PackageContentUriBuilder("https://unittest.org/packages/{id-lower}/{version-lower}.nupkg"); + + Assert.Throws(() => packageContentUriBuilder.Build(packageId, normalizedPackageVersion)); + } + + [Fact] + public void ProperlyBuildsPackageContentUrl() + { + // Arrange + var packageContentUriBuilder = new PackageContentUriBuilder("https://unittest.org/packages/{id-lower}/{version-lower}.nupkg"); + var expectedUrl = new Uri("https://unittest.org/packages/package.id/1.0.0-alpha.1.nupkg"); + + // Act + var actualUrl = packageContentUriBuilder.Build("Package.Id", "1.0.0-Alpha.1"); + + // Assert + Assert.Equal(expectedUrl, actualUrl); + } + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/PackageUtilityTests.cs b/tests/CatalogTests/Helpers/PackageUtilityTests.cs new file mode 100644 index 000000000..edf342f44 --- /dev/null +++ b/tests/CatalogTests/Helpers/PackageUtilityTests.cs @@ -0,0 +1,44 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Services.Metadata.Catalog.Helpers; +using Xunit; + +namespace CatalogTests.Helpers +{ + public class PackageUtilityTests + { + [Theory] + [InlineData(null)] + [InlineData("")] + public void GetPackageFileName_WhenPackageIdIsNullOrEmpty_Throws(string packageId) + { + var exception = Assert.Throws( + () => PackageUtility.GetPackageFileName(packageId, packageVersion: "a")); + + Assert.Equal("packageId", exception.ParamName); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public void GetPackageFileName_WhenPackageVersionIsNullOrEmpty_Throws(string packageVersion) + { + var exception = Assert.Throws( + () => PackageUtility.GetPackageFileName(packageId: "a", packageVersion: packageVersion)); + + Assert.Equal("packageVersion", exception.ParamName); + } + + [Theory] + [InlineData("a", "b")] + [InlineData("A", "B")] + public void GetPackageFileName_WithValidArguments_ReturnsFileName(string packageId, string packageVersion) + { + var packageFileName = PackageUtility.GetPackageFileName(packageId, packageVersion); + + Assert.Equal($"{packageId}.{packageVersion}.nupkg", packageFileName); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/RegistrationIndependentPackage.cs b/tests/CatalogTests/Helpers/RegistrationIndependentPackage.cs new file mode 100644 index 000000000..efe89f6a7 --- /dev/null +++ b/tests/CatalogTests/Helpers/RegistrationIndependentPackage.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class RegistrationIndependentPackage + { + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string[] TypeKeyword { get; } + [JsonProperty(CatalogConstants.CatalogEntry)] + internal string CatalogEntry { get; } + [JsonProperty(CatalogConstants.Listed)] + internal bool Listed { get; } + [JsonProperty(CatalogConstants.PackageContent)] + internal string PackageContent { get; } + [JsonProperty(CatalogConstants.Published)] + internal string Published { get; } + [JsonProperty(CatalogConstants.Registration)] + internal string Registration { get; } + [JsonProperty(CatalogConstants.ContextKeyword)] + internal JObject ContextKeyword { get; } + + [JsonConstructor] + internal RegistrationIndependentPackage( + string idKeyword, + string[] typeKeyword, + string catalogEntry, + bool listed, + string packageContent, + string published, + string registration, + JObject contextKeyword) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + CatalogEntry = catalogEntry; + Listed = listed; + PackageContent = packageContent; + Published = published; + Registration = registration; + ContextKeyword = contextKeyword; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/RegistrationIndependentPage.cs b/tests/CatalogTests/Helpers/RegistrationIndependentPage.cs new file mode 100644 index 000000000..ba0d26d4c --- /dev/null +++ b/tests/CatalogTests/Helpers/RegistrationIndependentPage.cs @@ -0,0 +1,32 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class RegistrationIndependentPage : RegistrationPage + { + [JsonProperty(CatalogConstants.ContextKeyword)] + internal JObject ContextKeyword { get; } + + [JsonConstructor] + internal RegistrationIndependentPage( + string idKeyword, + string typeKeyword, + string commitId, + string commitTimeStamp, + int count, + RegistrationPackage[] items, + string parent, + string lower, + string upper, + JObject contextKeyword) + : base(idKeyword, typeKeyword, commitId, commitTimeStamp, count, items, parent, lower, upper) + { + ContextKeyword = contextKeyword; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/RegistrationIndex.cs b/tests/CatalogTests/Helpers/RegistrationIndex.cs new file mode 100644 index 000000000..57f4a954b --- /dev/null +++ b/tests/CatalogTests/Helpers/RegistrationIndex.cs @@ -0,0 +1,46 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class RegistrationIndex + { + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string[] TypeKeyword { get; } + [JsonProperty(CatalogConstants.CommitId)] + internal string CommitId { get; } + [JsonProperty(CatalogConstants.CommitTimeStamp)] + internal string CommitTimeStamp { get; } + [JsonProperty(CatalogConstants.Count)] + internal int Count { get; } + [JsonProperty(CatalogConstants.Items)] + internal RegistrationPage[] Items { get; } + [JsonProperty(CatalogConstants.ContextKeyword)] + internal JObject ContextKeyword { get; } + + [JsonConstructor] + internal RegistrationIndex( + string idKeyword, + string[] typeKeyword, + string commitId, + string commitTimeStamp, + int count, + RegistrationPage[] items, + JObject contextKeyword) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + CommitId = commitId; + CommitTimeStamp = commitTimeStamp; + Count = count; + Items = items; + ContextKeyword = contextKeyword; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/RegistrationPackage.cs b/tests/CatalogTests/Helpers/RegistrationPackage.cs new file mode 100644 index 000000000..391629e4b --- /dev/null +++ b/tests/CatalogTests/Helpers/RegistrationPackage.cs @@ -0,0 +1,45 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class RegistrationPackage + { + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string TypeKeyword { get; } + [JsonProperty(CatalogConstants.CommitId)] + internal string CommitId { get; } + [JsonProperty(CatalogConstants.CommitTimeStamp)] + internal string CommitTimeStamp { get; } + [JsonProperty(CatalogConstants.CatalogEntry)] + internal RegistrationPackageDetails CatalogEntry { get; } + [JsonProperty(CatalogConstants.PackageContent)] + internal string PackageContent { get; } + [JsonProperty(CatalogConstants.Registration)] + internal string Registration { get; } + + [JsonConstructor] + internal RegistrationPackage( + string idKeyword, + string typeKeyword, + string commitId, + string commitTimeStamp, + RegistrationPackageDetails catalogEntry, + string packageContent, + string registration) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + CommitId = commitId; + CommitTimeStamp = commitTimeStamp; + CatalogEntry = catalogEntry; + PackageContent = packageContent; + Registration = registration; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/RegistrationPackageDeprecationAlternatePackage.cs b/tests/CatalogTests/Helpers/RegistrationPackageDeprecationAlternatePackage.cs new file mode 100644 index 000000000..e3ce467cf --- /dev/null +++ b/tests/CatalogTests/Helpers/RegistrationPackageDeprecationAlternatePackage.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NgTests; + +namespace CatalogTests.Helpers +{ + public class RegistrationPackageDeprecationAlternatePackage + { + [JsonConstructor] + public RegistrationPackageDeprecationAlternatePackage( + string id, + string range) + { + Id = id; + Range = range; + } + + [JsonProperty(CatalogConstants.Id)] + public string Id { get; } + + [JsonProperty(CatalogConstants.Range)] + public string Range { get; } + } +} diff --git a/tests/CatalogTests/Helpers/RegistrationPackageDeprecationDetails.cs b/tests/CatalogTests/Helpers/RegistrationPackageDeprecationDetails.cs new file mode 100644 index 000000000..f4ca7328a --- /dev/null +++ b/tests/CatalogTests/Helpers/RegistrationPackageDeprecationDetails.cs @@ -0,0 +1,31 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NgTests; + +namespace CatalogTests.Helpers +{ + public class RegistrationPackageDeprecation + { + [JsonConstructor] + public RegistrationPackageDeprecation( + string[] reasons, + string message = null, + RegistrationPackageDeprecationAlternatePackage alternatePackage = null) + { + Reasons = reasons; + Message = message; + AlternatePackage = alternatePackage; + } + + [JsonProperty(CatalogConstants.Reasons)] + public string[] Reasons { get; } + + [JsonProperty(CatalogConstants.Message)] + public string Message { get; } + + [JsonProperty(CatalogConstants.AlternatePackage)] + public RegistrationPackageDeprecationAlternatePackage AlternatePackage { get; } + } +} diff --git a/tests/CatalogTests/Helpers/RegistrationPackageDetails.cs b/tests/CatalogTests/Helpers/RegistrationPackageDetails.cs new file mode 100644 index 000000000..5a371c0cc --- /dev/null +++ b/tests/CatalogTests/Helpers/RegistrationPackageDetails.cs @@ -0,0 +1,93 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal sealed class RegistrationPackageDetails + { + [JsonProperty(CatalogConstants.IdKeyword)] + internal string IdKeyword { get; } + [JsonProperty(CatalogConstants.TypeKeyword)] + internal string TypeKeyword { get; } + [JsonProperty(CatalogConstants.Authors)] + internal string Authors { get; } + [JsonProperty(CatalogConstants.Deprecation)] + internal RegistrationPackageDeprecation Deprecation { get; } + [JsonProperty(CatalogConstants.Description)] + internal string Description { get; } + [JsonProperty(CatalogConstants.IconUrl)] + internal string IconUrl { get; } + [JsonProperty(CatalogConstants.Id)] + internal string Id { get; } + [JsonProperty(CatalogConstants.Language)] + internal string Language { get; } + [JsonProperty(CatalogConstants.LicenseUrl)] + internal string LicenseUrl { get; } + [JsonProperty(CatalogConstants.Listed)] + internal bool Listed { get; } + [JsonProperty(CatalogConstants.MinClientVersion)] + internal string MinClientVersion { get; } + [JsonProperty(CatalogConstants.PackageContent)] + internal string PackageContent { get; } + [JsonProperty(CatalogConstants.ProjectUrl)] + internal string ProjectUrl { get; } + [JsonProperty(CatalogConstants.Published)] + internal string Published { get; } + [JsonProperty(CatalogConstants.RequireLicenseAcceptance)] + internal bool RequireLicenseAcceptance { get; } + [JsonProperty(CatalogConstants.Summary)] + internal string Summary { get; } + [JsonProperty(CatalogConstants.Tags)] + internal string[] Tags { get; } + [JsonProperty(CatalogConstants.Title)] + internal string Title { get; } + [JsonProperty(CatalogConstants.Version)] + internal string Version { get; } + + [JsonConstructor] + internal RegistrationPackageDetails( + string idKeyword, + string typeKeyword, + string authors, + RegistrationPackageDeprecation deprecation, + string description, + string iconUrl, + string id, + string language, + string licenseUrl, + bool listed, + string minClientVersion, + string packageContent, + string projectUrl, + string published, + bool requireLicenseAcceptance, + string summary, + string[] tags, + string title, + string version) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + Authors = authors; + Deprecation = deprecation; + Description = description; + IconUrl = iconUrl; + Id = id; + Language = language; + LicenseUrl = licenseUrl; + Listed = listed; + MinClientVersion = minClientVersion; + PackageContent = packageContent; + ProjectUrl = projectUrl; + Published = published; + RequireLicenseAcceptance = requireLicenseAcceptance; + Summary = summary; + Tags = tags; + Title = title; + Version = version; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/RegistrationPage.cs b/tests/CatalogTests/Helpers/RegistrationPage.cs new file mode 100644 index 000000000..86c19f3d0 --- /dev/null +++ b/tests/CatalogTests/Helpers/RegistrationPage.cs @@ -0,0 +1,53 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Newtonsoft.Json; +using NgTests; + +namespace CatalogTests.Helpers +{ + internal class RegistrationPage + { + [JsonProperty(CatalogConstants.IdKeyword)] + public string IdKeyword { get; protected set; } + [JsonProperty(CatalogConstants.TypeKeyword)] + public string TypeKeyword { get; protected set; } + [JsonProperty(CatalogConstants.CommitId)] + public string CommitId { get; } + [JsonProperty(CatalogConstants.CommitTimeStamp)] + public string CommitTimeStamp { get; protected set; } + [JsonProperty(CatalogConstants.Count)] + public int Count { get; protected set; } + [JsonProperty(CatalogConstants.Items)] + public RegistrationPackage[] Items { get; protected set; } + [JsonProperty(CatalogConstants.Parent)] + public string Parent { get; protected set; } + [JsonProperty(CatalogConstants.Lower)] + public string Lower { get; protected set; } + [JsonProperty(CatalogConstants.Upper)] + public string Upper { get; protected set; } + + [JsonConstructor] + internal RegistrationPage( + string idKeyword, + string typeKeyword, + string commitId, + string commitTimeStamp, + int count, + RegistrationPackage[] items, + string parent, + string lower, + string upper) + { + IdKeyword = idKeyword; + TypeKeyword = typeKeyword; + CommitId = commitId; + CommitTimeStamp = commitTimeStamp; + Count = count; + Items = items; + Parent = parent; + Lower = lower; + Upper = upper; + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Helpers/UtilsTests.cs b/tests/CatalogTests/Helpers/UtilsTests.cs new file mode 100644 index 000000000..92d71487a --- /dev/null +++ b/tests/CatalogTests/Helpers/UtilsTests.cs @@ -0,0 +1,198 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using NuGet.Services.Metadata.Catalog; +using Xunit; +using VDS.RDF; + +namespace CatalogTests.Helpers +{ + public class UtilsTests + { + [Fact] + public void GetNupkgMetadata_WhenStreamIsNull_Throws() + { + var exception = Assert.Throws( + () => Utils.GetNupkgMetadata(stream: null, packageHash: "a")); + + Assert.Equal("stream", exception.ParamName); + } + + [Fact] + public void GetNupkgMetadata_WhenPackageHashIsNull_GeneratesPackageHash() + { + using (var stream = GetPackageStream()) + { + var metadata = Utils.GetNupkgMetadata(stream, packageHash: null); + + Assert.NotNull(metadata.Nuspec); + Assert.Equal(18, metadata.Entries.Count()); + Assert.Equal("bq5DjCtCJpy9R5rsEeQlKz8qGF1Bh3wGaJKMlRwmCoKZ8WUCIFtU3JlyMOdAkSn66KCehCCAxMZFOQD4nNnH/w==", metadata.PackageHash); + Assert.Equal(1871318, metadata.PackageSize); + } + } + + [Fact] + public void GetNupkgMetadata_WhenPackageHashIsProvided_UsesProvidePackageHash() + { + using (var stream = GetPackageStream()) + { + var metadata = Utils.GetNupkgMetadata(stream, packageHash: "a"); + + Assert.NotNull(metadata.Nuspec); + Assert.Equal(18, metadata.Entries.Count()); + Assert.Equal("a", metadata.PackageHash); + Assert.Equal(1871318, metadata.PackageSize); + } + } + + [Fact] + public void GetNupkgMetadata_WhenNuspecNotFound_Throws() + { + using (var stream = new MemoryStream()) + { + using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true)) + { + zipArchive.CreateEntry("a"); + } + + stream.Position = 0; + + var exception = Assert.Throws( + () => Utils.GetNupkgMetadata(stream, packageHash: null)); + + Assert.StartsWith("Unable to find nuspec", exception.Message); + } + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public void GetResourceStream_WhenResourceNameIsNullOrEmpty_Throws(string resourceName) + { + var exception = Assert.Throws(() => Utils.GetResourceStream(resourceName)); + + Assert.Equal("resourceName", exception.ParamName); + } + + [Fact] + public void GetResourceStream_WhenResourceNameIsValid_ReturnsStream() + { + using (var stream = Utils.GetResourceStream("context.Catalog.json")) + { + Assert.NotNull(stream); + } + } + + private static MemoryStream GetPackageStream() + { + return TestHelper.GetStream("Newtonsoft.Json.9.0.2-beta1.nupkg"); + } + + [Fact] + public void GetNupkgMetadataWithLicenseUrl_ReturnsLicenseUrl() + { + // Arrange + var stream = TestHelper.GetStream("Newtonsoft.Json.9.0.2-beta1.nupkg"); + var metadata = Utils.GetNupkgMetadata(stream, packageHash: null); + var baseUrl = "http://example/"; + var uriNodeName = new Uri(String.Concat(baseUrl, "newtonsoft.json.9.0.2-beta1.json")); + + // Act + var graph = Utils.CreateNuspecGraph(metadata.Nuspec, baseUrl, normalizeXml: true); + var licenseFileTriples = graph.GetTriplesWithSubjectPredicate( + graph.CreateUriNode(uriNodeName), + graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet + "licenseFile")))); + var licenseExpressionTriples = graph.GetTriplesWithSubjectPredicate( + graph.CreateUriNode(uriNodeName), + graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet + "licenseExpression")))); + var licenseUrlTriples = graph.GetTriplesWithSubjectPredicate( + graph.CreateUriNode(uriNodeName), + graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet + "licenseUrl")))); + + // Assert + Assert.Empty(licenseFileTriples); + Assert.Empty(licenseExpressionTriples); + Assert.Single(licenseUrlTriples); + } + + [Theory] + [InlineData("TestPackage.LicenseExpression.0.1.0.nupkg", "licenseExpression", "MIT", 0)] + [InlineData("TestPackage.LicenseFile.0.1.0.nupkg", "licenseFile", "license.txt", 0)] + [InlineData("TestPackage.LicenseExpressionAndUrl.0.1.0.nupkg", "licenseExpression", "MIT", 1)] + [InlineData("TestPackage.LicenseFileAndUrl.0.1.0.nupkg", "licenseFile", "license.txt", 1)] + public void GetNupkgMetadataWithLicenseType_ReturnsLicense(string packageName, string licenseType, string licenseContent, int expectedLicenseUrlNumber) + { + // Arrange + var stream = TestHelper.GetStream(packageName); + var metadata = Utils.GetNupkgMetadata(stream, packageHash: null); + var baseUrl = "http://example/"; + var uriNodeName = new Uri(String.Concat(baseUrl, "testpackage.license.0.1.0.json")); + + // Act + var graph = Utils.CreateNuspecGraph(metadata.Nuspec, baseUrl, normalizeXml: true); + var licenseTriples = graph.GetTriplesWithSubjectPredicate( + graph.CreateUriNode(uriNodeName), + graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet, licenseType)))); + var licenseUrlTriples = graph.GetTriplesWithSubjectPredicate( + graph.CreateUriNode(uriNodeName), + graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet + "licenseUrl")))); + var result = (LiteralNode)licenseTriples.First().Object; + + // Assert + Assert.Equal(expectedLicenseUrlNumber, licenseUrlTriples.Count()); + Assert.Single(licenseTriples); + Assert.Equal(licenseContent, result.Value); + } + + [Theory] + [InlineData("TestPackage.IconAndIconUrl.0.4.2.nupkg", true, true)] + [InlineData("TestPackage.IconOnlyEmptyType.0.4.2.nupkg", false, false)] + [InlineData("TestPackage.IconOnlyFileType.0.4.2.nupkg", true, false)] + [InlineData("TestPackage.IconOnlyInvalidType.0.4.2.nupkg", false, false)] + [InlineData("TestPackage.IconOnlyNoType.0.4.2.nupkg", true, false)] + public void GetNupkgMetadataWithIcon_ProcessesCorrectly(string packageFilename, bool expectedIconMetadata, bool expectedIconUrlMetadata) + { + // Arrange + var stream = TestHelper.GetStream(packageFilename); + var metadata = Utils.GetNupkgMetadata(stream, packageHash: null); + var baseUrl = "http://example/"; + var packageIdVersion = packageFilename.Replace(".nupkg", "").ToLowerInvariant(); + var uriNodeName = new Uri(string.Concat(baseUrl, packageIdVersion, ".json")); + + // Act + var graph = Utils.CreateNuspecGraph(metadata.Nuspec, baseUrl, normalizeXml: true); + var iconTriples = graph.GetTriplesWithSubjectPredicate( + graph.CreateUriNode(uriNodeName), + graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet + "iconFile")))); + var iconUrlTriples = graph.GetTriplesWithSubjectPredicate( + graph.CreateUriNode(uriNodeName), + graph.CreateUriNode(new Uri(String.Concat(Schema.Prefixes.NuGet + "iconUrl")))); + var result = (LiteralNode)iconTriples.FirstOrDefault()?.Object; + + // Assert + if (expectedIconMetadata) + { + Assert.Single(iconTriples); + Assert.Equal("icon.png", result.Value); + } + else + { + Assert.Empty(iconTriples); + } + + if (expectedIconUrlMetadata) + { + Assert.Single(iconUrlTriples); + } + else + { + Assert.Empty(iconUrlTriples); + } + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Icons/CatalogLeafDataProcessorFacts.cs b/tests/CatalogTests/Icons/CatalogLeafDataProcessorFacts.cs new file mode 100644 index 000000000..0c61657ce --- /dev/null +++ b/tests/CatalogTests/Icons/CatalogLeafDataProcessorFacts.cs @@ -0,0 +1,623 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.WindowsAzure.Storage; +using Moq; +using Moq.Protected; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Icons; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Versioning; +using Xunit; + +namespace CatalogTests.Icons +{ + public class CatalogLeafDataProcessorFacts + { + public class TheProcessPackageDeleteLeafAsyncMethod : TestBase + { + [Theory] + [InlineData("Package", "1.2.3-Preview", "package/1.2.3-preview/icon")] + [InlineData("PpPpPPpP", "3.2.1+metaData", "pppppppp/3.2.1/icon")] + [InlineData("Package", "01.2.03.4", "package/1.2.3.4/icon")] + public async Task CallsDeleteIconProperly(string packageId, string packageVersion, string expectedPath) + { + var leaf = CreateCatalogLeaf(packageId, packageVersion); + + var destinationStorageMock = new Mock(new Uri("https://base/storage")); + + await Target.ProcessPackageDeleteLeafAsync(destinationStorageMock.Object, leaf, CancellationToken.None); + + IconProcessorMock + .Verify(ip => ip.DeleteIconAsync(destinationStorageMock.Object, expectedPath, CancellationToken.None, It.IsAny(), It.IsAny())); + } + } + + public class TheProcessPackageDetailsLeafAsyncMethod : TestBase + { + private const string IconUrlString = "https://icon/url"; + private const string CachedResult = "https://cached/result"; + + [Theory] + [InlineData("Package", "1.2.3-Preview", false, "package/1.2.3-preview/icon")] + [InlineData("PpPpPPpP", "3.2.1+metaData", false, "pppppppp/3.2.1/icon")] + [InlineData("Package", "01.2.03.4", false, "package/1.2.3.4/icon")] + [InlineData("Package", "1.2.3-Preview", true, "package/1.2.3-preview/icon")] + [InlineData("PpPpPPpP", "3.2.1+metaData", true, "pppppppp/3.2.1/icon")] + [InlineData("Package", "01.2.03.4", true, "package/1.2.3.4/icon")] + public async Task CallsCopyEmbeddedIconFromPackageProperly(string packageId, string packageVersion, bool hasIconUrl, string expectedPath) + { + var leaf = CreateCatalogLeaf(packageId, packageVersion); + + const string iconFilename = "iconFilename"; + + await Target.ProcessPackageDetailsLeafAsync( + DestinationStorageMock.Object, + IconCacheStorageMock.Object, + leaf, + hasIconUrl ? IconUrlString : null, + iconFilename, + CancellationToken.None); + + IconProcessorMock + .Verify( + ip => ip.CopyEmbeddedIconFromPackageAsync( + It.IsAny(), + iconFilename, + DestinationStorageMock.Object, + expectedPath, + CancellationToken.None, + It.Is(p => leaf.PackageIdentity.Id.Equals(p, StringComparison.OrdinalIgnoreCase)), + It.Is(v => leaf.PackageIdentity.Version.ToNormalizedString().Equals(v, StringComparison.OrdinalIgnoreCase))), + Times.Once); + } + + [Theory] + [InlineData("Package", "1.2.3-Preview", "package/1.2.3-preview/icon")] + [InlineData("PpPpPPpP", "3.2.1+metaData", "pppppppp/3.2.1/icon")] + [InlineData("Package", "01.2.03.4", "package/1.2.3.4/icon")] + public async Task CopiesIconFromExternalLocation(string packageId, string packageVersion, string expectedPath) + { + var leaf = CreateCatalogLeaf(packageId, packageVersion); + + ExternalIconContentProviderMock + .Setup(cp => cp.TryGetResponseAsync( + It.Is(u => u.AbsoluteUri == IconUrlString), + CancellationToken.None)) + .ReturnsAsync( + TryGetResponseResult.Success( + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = ExternalIconContentMock.Object + })); + + await Target.ProcessPackageDetailsLeafAsync( + DestinationStorageMock.Object, + IconCacheStorageMock.Object, + leaf, + IconUrlString, + null, + CancellationToken.None); + + IconProcessorMock + .Verify( + ip => ip.CopyIconFromExternalSourceAsync( + ExternalIconStream, + DestinationStorageMock.Object, + expectedPath, + CancellationToken.None, + packageId, + leaf.PackageIdentity.Version.ToNormalizedString()), + Times.Once); + } + + [Fact] + public async Task IgnoresPackageNotFoundExceptions() + { + var leaf = CreateCatalogLeaf("packageid", "1.2.3"); + var packageUri = new Uri("https://package/url"); + var cloudBlockBlobMock = new Mock(); + PackageStorageMock + .Setup(ps => ps.ResolveUri(It.IsAny())) + .Returns(packageUri); + PackageStorageMock + .Setup(ps => ps.GetCloudBlockBlobReferenceAsync(packageUri)) + .ReturnsAsync(cloudBlockBlobMock.Object); + var exception = new StorageException(new RequestResult { HttpStatusCode = 404 }, message: "Exception!!1", inner: null); + cloudBlockBlobMock + .Setup(cbb => cbb.GetStreamAsync(It.IsAny())) + .ThrowsAsync(exception); + + await Target.ProcessPackageDetailsLeafAsync( + DestinationStorageMock.Object, + IconCacheStorageMock.Object, + leaf, + null, + "icon.png", + CancellationToken.None); + + IconProcessorMock + .Verify(ip => ip.CopyEmbeddedIconFromPackageAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), Times.Never); + } + + [Fact] + public async Task RetriesExternalLocationFailures() + { + var leaf = CreateCatalogLeaf(); + + ExternalIconContentProviderMock + .SetupSequence(cp => cp.TryGetResponseAsync( + It.Is(u => u.AbsoluteUri == IconUrlString), + CancellationToken.None)) + .ReturnsAsync(TryGetResponseResult.FailCanRetry()) + .ReturnsAsync( + TryGetResponseResult.Success( + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = ExternalIconContentMock.Object + })); + + await Target.ProcessPackageDetailsLeafAsync( + DestinationStorageMock.Object, + IconCacheStorageMock.Object, + leaf, + IconUrlString, + null, + CancellationToken.None); + + ExternalIconContentProviderMock + .Verify( + cp => cp.TryGetResponseAsync( + It.Is(u => u.AbsoluteUri == IconUrlString), + CancellationToken.None), + Times.AtLeast(2)); + + IconProcessorMock + .Verify( + ip => ip.CopyIconFromExternalSourceAsync( + ExternalIconStream, + DestinationStorageMock.Object, + "theid/3.4.2/icon", + CancellationToken.None, + "theid", + leaf.PackageIdentity.Version.ToNormalizedString()), + Times.Once); + } + + [Fact] + public async Task DoesNotRetrySeriousFailures() + { + var leaf = CreateCatalogLeaf(); + + ExternalIconContentProviderMock + .SetupSequence(cp => cp.TryGetResponseAsync( + It.Is(u => u.AbsoluteUri == IconUrlString), + CancellationToken.None)) + .ReturnsAsync(TryGetResponseResult.FailCannotRetry()) + .ReturnsAsync( + TryGetResponseResult.Success( + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = ExternalIconContentMock.Object + })); + + await Target.ProcessPackageDetailsLeafAsync( + DestinationStorageMock.Object, + IconCacheStorageMock.Object, + leaf, + IconUrlString, + null, + CancellationToken.None); + + ExternalIconContentProviderMock + .Verify( + cp => cp.TryGetResponseAsync( + It.Is(u => u.AbsoluteUri == IconUrlString), + CancellationToken.None), + Times.AtMostOnce); + + VerifyNoCopyFromExternalSource(); + } + + [Fact] + public async Task TriesToGetFromCache() + { + var leaf = CreateCatalogLeaf(); + + IconCopyResultCacheMock + .Setup(c => c.Get(It.Is(u => u.AbsoluteUri == IconUrlString))) + .Returns(ExternalIconCopyResult.Success(new Uri(IconUrlString), new Uri(CachedResult))); + IconCacheStorageMock + .Setup( + ds => ds.CopyAsync( + It.Is(u => u.AbsoluteUri == CachedResult), + DestinationStorageMock.Object, + It.Is(u => u.AbsoluteUri == ResolvedUriString), + It.IsAny>(), + CancellationToken.None)) + .Returns(Task.CompletedTask); + + await Target.ProcessPackageDetailsLeafAsync( + DestinationStorageMock.Object, + IconCacheStorageMock.Object, + leaf, + IconUrlString, + null, + CancellationToken.None); + + IconCacheStorageMock + .Verify( + ds => ds.CopyAsync( + It.Is(u => u.AbsoluteUri == CachedResult), + DestinationStorageMock.Object, + It.Is(u => u.AbsoluteUri == ResolvedUriString), + It.IsAny>(), + CancellationToken.None), + Times.Once); + + VerifyNoCopyFromExternalSource(); + } + + [Fact] + public async Task DoesNotTryToCopyPrevioslyFailedIcons() + { + var leaf = CreateCatalogLeaf(); + + IconCopyResultCacheMock + .Setup(c => c.Get(It.Is(u => u.AbsoluteUri == IconUrlString))) + .Returns(ExternalIconCopyResult.Fail(new Uri(IconUrlString), TimeSpan.FromMinutes(1))); + + await Target.ProcessPackageDetailsLeafAsync( + DestinationStorageMock.Object, + IconCacheStorageMock.Object, + leaf, + IconUrlString, + null, + CancellationToken.None); + + DestinationStorageMock + .Verify( + ds => ds.CopyAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny()), + Times.Never); + + VerifyNoCopyFromExternalSource(); + } + + [Fact] + public async Task DeletesIconOnCopyFailure() + { + var leaf = CreateCatalogLeaf(); + + ExternalIconContentProviderMock + .Setup(cp => cp.TryGetResponseAsync( + It.Is(u => u.AbsoluteUri == IconUrlString), + CancellationToken.None)) + .ReturnsAsync(TryGetResponseResult.FailCannotRetry()); + + await Target.ProcessPackageDetailsLeafAsync( + DestinationStorageMock.Object, + IconCacheStorageMock.Object, + leaf, + IconUrlString, + null, + CancellationToken.None); + + IconProcessorMock + .Verify(ip => ip.DeleteIconAsync(DestinationStorageMock.Object, "theid/3.4.2/icon", CancellationToken.None, leaf.PackageIdentity.Id, leaf.PackageIdentity.Version.ToNormalizedString())); + } + + [Fact] + public async Task DeletesIconOnCachedCopyFailure() + { + var leaf = CreateCatalogLeaf(); + + IconCopyResultCacheMock + .Setup(c => c.Get(It.Is(u => u.AbsoluteUri == IconUrlString))) + .Returns(ExternalIconCopyResult.Fail(new Uri(IconUrlString), TimeSpan.FromHours(1))); + + await Target.ProcessPackageDetailsLeafAsync( + DestinationStorageMock.Object, + IconCacheStorageMock.Object, + leaf, + IconUrlString, + null, + CancellationToken.None); + + IconProcessorMock + .Verify(ip => ip.DeleteIconAsync(DestinationStorageMock.Object, "theid/3.4.2/icon", CancellationToken.None, leaf.PackageIdentity.Id, leaf.PackageIdentity.Version.ToNormalizedString())); + } + + [Fact] + public async Task RetriesFailedCopyFromOperationsCache() + { + var leaf = CreateCatalogLeaf(); + + IconCopyResultCacheMock + .Setup(c => c.Get(It.Is(u => u.AbsoluteUri == IconUrlString))) + .Returns(ExternalIconCopyResult.Success(new Uri(IconUrlString), new Uri(CachedResult))); + + IconCacheStorageMock + .SetupSequence( + ds => ds.CopyAsync( + It.Is(u => u.AbsoluteUri == CachedResult), + DestinationStorageMock.Object, + It.Is(u => u.AbsoluteUri == ResolvedUriString), + It.IsAny>(), + CancellationToken.None)) + .Throws(new Exception("Core meltdown")) + .Returns(Task.CompletedTask); + + await Target.ProcessPackageDetailsLeafAsync( + DestinationStorageMock.Object, + IconCacheStorageMock.Object, + leaf, + IconUrlString, + null, + CancellationToken.None); + + IconCacheStorageMock + .Verify( + ds => ds.CopyAsync( + It.Is(u => u.AbsoluteUri == CachedResult), + DestinationStorageMock.Object, + It.Is(u => u.AbsoluteUri == ResolvedUriString), + It.IsAny>(), + CancellationToken.None), + Times.AtLeast(2)); + + VerifyNoCopyFromExternalSource(); + } + + [Fact] + public async Task FallsBackToRetrievingFromExternalStoreIfCopyFromCacheFails() + { + var leaf = CreateCatalogLeaf(); + + IconCopyResultCacheMock + .Setup(c => c.Get(It.Is(u => u.AbsoluteUri == IconUrlString))) + .Returns(ExternalIconCopyResult.Success(new Uri(IconUrlString), new Uri(CachedResult))); + + IconCacheStorageMock + .Setup( + ds => ds.CopyAsync( + It.Is(u => u.AbsoluteUri == CachedResult), + DestinationStorageMock.Object, + It.Is(u => u.AbsoluteUri == ResolvedUriString), + It.IsAny>(), + CancellationToken.None)) + .Throws(new Exception("Core meltdown")); + + ExternalIconContentProviderMock + .Setup(cp => cp.TryGetResponseAsync( + It.Is(u => u.AbsoluteUri == IconUrlString), + CancellationToken.None)) + .ReturnsAsync( + TryGetResponseResult.Success( + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = ExternalIconContentMock.Object + })); + + await Target.ProcessPackageDetailsLeafAsync( + DestinationStorageMock.Object, + IconCacheStorageMock.Object, + leaf, + IconUrlString, + null, + CancellationToken.None); + + IconCacheStorageMock + .Verify( + ds => ds.CopyAsync( + It.Is(u => u.AbsoluteUri == CachedResult), + DestinationStorageMock.Object, + It.Is(u => u.AbsoluteUri == ResolvedUriString), + It.IsAny>(), + CancellationToken.None), + Times.AtLeast(2)); + + IconProcessorMock + .Verify( + ip => ip.CopyIconFromExternalSourceAsync( + ExternalIconStream, + DestinationStorageMock.Object, + "theid/3.4.2/icon", + CancellationToken.None, + "theid", + leaf.PackageIdentity.Version.ToNormalizedString()), + Times.Once); + } + + private Mock PackageBlobRerenceMock { get; set; } + private Stream ExternalIconStream { get; set; } + private Mock ExternalIconContentMock { get; set; } + + public TheProcessPackageDetailsLeafAsyncMethod() + { + PackageBlobRerenceMock = new Mock(); + + PackageStorageMock + .Setup(ps => ps.GetCloudBlockBlobReferenceAsync(It.IsAny())) + .ReturnsAsync(PackageBlobRerenceMock.Object); + + PackageBlobRerenceMock + .Setup(pbr => pbr.GetStreamAsync(It.IsAny())) + .ReturnsAsync(Mock.Of()); + + ExternalIconContentMock = new Mock(); + ExternalIconContentMock + .Protected() + .Setup>("CreateContentReadStreamAsync") + .ReturnsAsync(() => ExternalIconStream); + + ExternalIconStream = new MemoryStream(); + } + + private void VerifyNoCopyFromExternalSource() + { + IconProcessorMock + .Verify( + ip => ip.CopyIconFromExternalSourceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + } + + public class TestBase + { + protected const string ResolvedUriString = "https://resolved/destination/url"; + + protected Mock PackageStorageMock { get; set; } + protected Mock IconProcessorMock { get; set; } + protected Mock ExternalIconContentProviderMock { get; set; } + protected Mock IconCopyResultCacheMock { get; set; } + protected Mock TelemetryServiceMock { get; set; } + protected Mock> LoggerMock { get; set; } + protected Mock DestinationStorageMock { get; set; } + protected Mock IconCacheStorageMock { get; set; } + protected CatalogLeafDataProcessor Target { get; set; } + + public TestBase() + { + PackageStorageMock = new Mock(); + IconProcessorMock = new Mock(); + ExternalIconContentProviderMock = new Mock(); + IconCopyResultCacheMock = new Mock(); + TelemetryServiceMock = new Mock(); + LoggerMock = new Mock>(); + DestinationStorageMock = new Mock(); + IconCacheStorageMock = new Mock(); + + TelemetryServiceMock + .Setup(ts => ts.TrackEmbeddedIconProcessingDuration(It.IsAny(), It.IsAny())) + .Returns(Mock.Of()); + TelemetryServiceMock + .Setup(ts => ts.TrackExternalIconProcessingDuration(It.IsAny(), It.IsAny())) + .Returns(Mock.Of()); + + DestinationStorageMock + .Setup(ds => ds.ResolveUri(It.IsAny())) + .Returns(new Uri(ResolvedUriString)); + + Target = new CatalogLeafDataProcessor( + PackageStorageMock.Object, + IconProcessorMock.Object, + ExternalIconContentProviderMock.Object, + IconCopyResultCacheMock.Object, + TelemetryServiceMock.Object, + LoggerMock.Object); + } + + protected static CatalogCommitItem CreateCatalogLeaf(string packageId = "theid", string packageVersion = "3.4.2") + { + return new CatalogCommitItem( + new Uri("https://nuget.test/something"), + "somecommitid", + DateTime.UtcNow, + new string[0], + new Uri[0], + new PackageIdentity(packageId, new NuGetVersion(packageVersion))); + } + } + + private class TestStorage : Storage + { + private Func, CancellationToken, Task> _onCopy; + private Func _onDelete; + private Func> _onLoad; + + public TestStorage() + : base(new Uri("https://base/container")) + { + + } + + public override bool Exists(string fileName) + { + throw new NotImplementedException(); + } + + public override Task> ListAsync(CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + public void OnCopy(Func, CancellationToken, Task> callback) + { + _onCopy = callback; + } + + protected override async Task OnCopyAsync(Uri sourceUri, IStorage destinationStorage, Uri destinationUri, IReadOnlyDictionary destinationProperties, CancellationToken cancellationToken) + { + if (_onCopy != null) + { + await _onCopy(sourceUri, destinationStorage, destinationUri, destinationProperties, cancellationToken); + } + } + + public void OnDelete(Func callback) + { + _onDelete = callback; + } + + protected override async Task OnDeleteAsync(Uri resourceUri, DeleteRequestOptions deleteRequestOptions, CancellationToken cancellationToken) + { + if (_onDelete != null) + { + await _onDelete(resourceUri, deleteRequestOptions, cancellationToken); + } + } + + public void onLoad(Func> callback) + { + _onLoad = callback; + } + + protected override async Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken) + { + if (_onLoad != null) + { + return await _onLoad(resourceUri, cancellationToken); + } + + return null; + } + + public void OnSave(Func callback) + { + + } + + protected override Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + } + } +} diff --git a/tests/CatalogTests/Icons/IconCopyResultCacheFacts.cs b/tests/CatalogTests/Icons/IconCopyResultCacheFacts.cs new file mode 100644 index 000000000..f40a73529 --- /dev/null +++ b/tests/CatalogTests/Icons/IconCopyResultCacheFacts.cs @@ -0,0 +1,298 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using Newtonsoft.Json; +using NuGet.Services.Metadata.Catalog.Icons; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests.Icons +{ + + public class IconCopyResultCacheFacts + { + [Fact] + public async Task InitializeAsyncReadsJson() + { + Data = "{ \"https://key\": { \"SourceUrl\": \"https://source.test/data\", \"StorageUrl\": \"https://dest.test/data2\", \"IsCopySucceeded\": true } }"; + + await Target.InitializeAsync(CancellationToken.None); + + StorageMock.VerifyAll(); + var cache = Target.Get(new Uri("https://key")); + Assert.NotNull(cache); + Assert.Equal("https://source.test/data", cache.SourceUrl.AbsoluteUri); + Assert.Equal("https://dest.test/data2", cache.StorageUrl.AbsoluteUri); + } + + [Fact] + public void GetThrowsIfNotInitialized() + { + Assert.Throws(() => Target.Get(new Uri("https://whatever"))); + } + + [Fact] + public async Task SaveExternalIconThrowsIfNotInitialized() + { + await Assert.ThrowsAsync(() => Target.SaveExternalIcon(new Uri("https://whatever"), new Uri("https://storage.test"), Mock.Of(), Mock.Of(), CancellationToken.None)); + } + + [Fact] + public void SaveExternalCopyFailureThrowsIfNotInitialized() + { + Assert.Throws(() => Target.SaveExternalCopyFailure(new Uri("https://whatever"))); + } + + [Fact] + public void ClearThrowsIfNotInitialized() + { + Assert.Throws(() => Target.Clear(new Uri("https://whatever"))); + } + + [Theory] + [InlineData("https://source/d", null, false)] + [InlineData("https://source2/d", "https://dest/d", true)] + public async Task SmokeTest(string sourceUrl, string storageUrlString, bool expectedSuccess) + { + Data = "{}"; + await Target.InitializeAsync(CancellationToken.None); + + var storageUrl = storageUrlString == null ? null : new Uri(storageUrlString); + var success = storageUrlString != null; + + if (success) + { + await Target.SaveExternalIcon(new Uri(sourceUrl), storageUrl, StorageMock.Object, IconCacheStorageMock.Object, CancellationToken.None); + StorageMock + .Verify(ics => ics.CopyAsync(storageUrl, IconCacheStorageMock.Object, It.IsAny(), It.IsAny>(), CancellationToken.None)); + } + else + { + Target.SaveExternalCopyFailure(new Uri(sourceUrl)); + } + var item = Target.Get(new Uri(sourceUrl)); + Assert.Equal(sourceUrl, item.SourceUrl.AbsoluteUri); + if (success) + { + Assert.True(item.IsCopySucceeded); + Assert.Equal(DefaultResolvedUrlString, item.StorageUrl.AbsoluteUri); + } + else + { + Assert.False(item.IsCopySucceeded); + Assert.Null(item.StorageUrl); + } + Assert.Equal(expectedSuccess, item.IsCopySucceeded); + + Target.Clear(new Uri(sourceUrl)); + item = Target.Get(new Uri(sourceUrl)); + Assert.Null(item); + } + + [Fact] + public async Task SaveExternalIconDoesNotOverwriteSuccess() + { + Data = "{}"; + + await Target.InitializeAsync(CancellationToken.None); + + const string originalIconUrlString = "https://source/"; + var originalIconUrl = new Uri(originalIconUrlString); + const string firstSuccessStorageUrlString = "https://storage1/d"; + var firstSucessStorageUrl = new Uri(firstSuccessStorageUrlString); + var secondSuccessStorageUrl = new Uri("https://storage2"); + + await Target.SaveExternalIcon(originalIconUrl, firstSucessStorageUrl, StorageMock.Object, IconCacheStorageMock.Object, CancellationToken.None); + StorageMock + .Verify( + ics => ics.CopyAsync(firstSucessStorageUrl, IconCacheStorageMock.Object, It.IsAny(), It.IsAny>(), CancellationToken.None), + Times.Once); + await Target.SaveExternalIcon(originalIconUrl, secondSuccessStorageUrl, StorageMock.Object, IconCacheStorageMock.Object, CancellationToken.None); + StorageMock + .Verify( + ics => ics.CopyAsync(secondSuccessStorageUrl, It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny()), + Times.Never); + + var item = Target.Get(originalIconUrl); + Assert.True(item.IsCopySucceeded); + Assert.Equal(originalIconUrlString, item.SourceUrl.AbsoluteUri); + Assert.Equal(DefaultResolvedUrlString, item.StorageUrl.AbsoluteUri); + } + + [Fact] + public async Task SaveExternalIconOverwritesFailures() + { + Data = "{}"; + + await Target.InitializeAsync(CancellationToken.None); + + const string originalIconUrlString = "https://source/"; + var originalIconUrl = new Uri(originalIconUrlString); + const string successStorageUrlString = "https://storage2/"; + var successStorageUrl = new Uri(successStorageUrlString); + + Target.SaveExternalCopyFailure(originalIconUrl); + await Target.SaveExternalIcon(originalIconUrl, successStorageUrl, StorageMock.Object, IconCacheStorageMock.Object, CancellationToken.None); + StorageMock + .Verify( + ics => ics.CopyAsync(successStorageUrl, IconCacheStorageMock.Object, It.IsAny(), It.IsAny>(), CancellationToken.None), + Times.Once); + + var item = Target.Get(originalIconUrl); + + Assert.True(item.IsCopySucceeded); + Assert.Equal(originalIconUrlString, item.SourceUrl.AbsoluteUri); + Assert.Equal(DefaultResolvedUrlString, item.StorageUrl.AbsoluteUri); + } + + [Fact] + public async Task SavesCache() + { + Data = "{}"; + + await Target.InitializeAsync(CancellationToken.None); + + await Target.SaveExternalIcon(new Uri("https://sourcez"), new Uri("https://storage1/d"), StorageMock.Object, IconCacheStorageMock.Object, CancellationToken.None); + Target.SaveExternalCopyFailure(new Uri("https://sourcey")); + + string savedContent = null; + + StorageMock + .Setup(s => s.SaveAsync(new Uri("https://cache.test/blob"), It.IsAny(), CancellationToken.None)) + .Callback((_1, sc, _2) => savedContent = ((StringStorageContent)sc).Content) + .Returns(Task.CompletedTask); + + await Target.SaveAsync(CancellationToken.None); + + StorageMock + .Verify(s => s.SaveAsync(new Uri("https://cache.test/blob"), It.IsAny(), CancellationToken.None), Times.Once); + + Assert.Contains("https://sourcez", savedContent); + Assert.DoesNotContain("https://storage1/d", savedContent); // this url is only used for copying blob + Assert.Contains(DefaultResolvedUrlString, savedContent); + Assert.Contains("https://sourcey", savedContent); + } + + [Fact] + public async Task IgnoresFailuresWithoutExpiration() + { + const string iconUrl = "https://icon.test/url"; + Data = $"{{ \"{iconUrl}\": {{ \"SourceUrl\": \"{iconUrl}\", \"StorageUrl\": null }} }}"; + await Target.InitializeAsync(CancellationToken.None); + + var result = Target.Get(new Uri(iconUrl)); + + Assert.Null(result); + } + + [Fact] + public async Task IgnoresFailuresWithNullExpiration() + { + const string iconUrl = "https://icon.test/url"; + Data = $"{{ \"{iconUrl}\": {{ \"SourceUrl\": \"{iconUrl}\", \"StorageUrl\": null, \"Expiration\": null }} }}"; + await Target.InitializeAsync(CancellationToken.None); + + var result = Target.Get(new Uri(iconUrl)); + + Assert.Null(result); + } + + [Fact] + public async Task SavesFailureExpiration() + { + Data = "{}"; + + await Target.InitializeAsync(CancellationToken.None); + var time = DateTimeOffset.UtcNow; + const string failedUrl = "https://icon.test/fail"; + Target.SaveExternalCopyFailure(new Uri(failedUrl)); + + string savedContent = null; + + StorageMock + .Setup(s => s.SaveAsync(new Uri("https://cache.test/blob"), It.IsAny(), CancellationToken.None)) + .Callback((_1, sc, _2) => savedContent = ((StringStorageContent)sc).Content) + .Returns(Task.CompletedTask); + + await Target.SaveAsync(CancellationToken.None); + + var savedDictionary = JsonConvert.DeserializeObject>(savedContent); + var item = Assert.Single(savedDictionary, e => e.Key.AbsoluteUri == failedUrl); + Assert.NotNull(item.Value.Expiration); + Assert.True(item.Value.Expiration - time < TimeSpan.FromSeconds(2)); + } + + [Fact] + public async Task IgnoresExpiredFailures() + { + const string iconUrl = "https://icon.test/url"; + var expiration = DateTimeOffset.UtcNow.AddDays(-1).ToString("O"); + Data = $"{{ \"{iconUrl}\": {{ \"SourceUrl\": \"{iconUrl}\", \"StorageUrl\": null, \"Expiration\": \"{expiration}\" }} }}"; + await Target.InitializeAsync(CancellationToken.None); + + var result = Target.Get(new Uri(iconUrl)); + + Assert.Null(result); + } + + [Fact] + public async Task UsesUnexpiredFailures() + { + const string iconUrl = "https://icon.test/url"; + var expiration = DateTimeOffset.UtcNow.AddDays(1).ToString("O"); + Data = $"{{ \"{iconUrl}\": {{ \"SourceUrl\": \"{iconUrl}\", \"StorageUrl\": null, \"Expiration\": \"{expiration}\" }} }}"; + await Target.InitializeAsync(CancellationToken.None); + + var result = Target.Get(new Uri(iconUrl)); + + Assert.NotNull(result); + Assert.False(result.IsCopySucceeded); + } + + public IconCopyResultCacheFacts() + { + StorageMock = new Mock(); + + StorageMock + .Setup(s => s.ResolveUri("c2i_cache.json")) + .Returns(new Uri("https://cache.test/blob")) + .Verifiable(); + + IconCacheStorageMock = new Mock(); + IconCacheStorageMock + .Setup(ics => ics.ResolveUri(It.IsAny())) + .Returns(new Uri(DefaultResolvedUrlString)); + + var responseStreamContentMock = new Mock(); + responseStreamContentMock + .Setup(rsc => rsc.GetContentStream()) + .Returns(() => new MemoryStream(Encoding.UTF8.GetBytes(Data))) + .Verifiable(); + + StorageMock + .Setup(s => s.LoadAsync(new Uri("https://cache.test/blob"), It.IsAny())) + .ReturnsAsync(responseStreamContentMock.Object); + + Target = new IconCopyResultCache( + StorageMock.Object, + TimeSpan.FromMilliseconds(750), + Mock.Of>()); + } + + private const string DefaultResolvedUrlString = "https://resolved.test/uri"; + + private string Data { get; set; } + + private Mock StorageMock { get; set; } + private IconCopyResultCache Target { get; set; } + private Mock IconCacheStorageMock { get; set; } + } +} diff --git a/tests/CatalogTests/Icons/IconProcessorFacts.cs b/tests/CatalogTests/Icons/IconProcessorFacts.cs new file mode 100644 index 000000000..c26e29f05 --- /dev/null +++ b/tests/CatalogTests/Icons/IconProcessorFacts.cs @@ -0,0 +1,161 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Icons; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests.Icons +{ + public class IconProcessorFacts + { + public class TheCopyIconFromExternalSourceMethod : TestBase + { + [Fact] + public async Task ReadsAndUsesStreamData() + { + var data = new byte[] { 0xFF, 0xD8, 0xFF, 0xAA }; + + using (var ms = new MemoryStream(data)) + { + await Target.CopyIconFromExternalSourceAsync(ms, DestinationStorageMock.Object, "somePath", CancellationToken.None, "theid", "1.2.3"); + } + + DestinationStorageMock.Verify(ds => ds.SaveAsync( + It.IsAny(), + It.Is(sc => SameData(data, sc)), + It.IsAny())); + } + + [Theory] + [MemberData(nameof(ImageData))] + public async Task DeterminesContentType(byte[] data, string expectedContentType) + { + using (var ms = new MemoryStream(data)) + { + await Target.CopyIconFromExternalSourceAsync(ms, DestinationStorageMock.Object, "somePath", CancellationToken.None, "theid", "1.2.3"); + } + + DestinationStorageMock.Verify(ds => ds.SaveAsync( + It.IsAny(), + It.Is(sc => expectedContentType == sc.ContentType), + It.IsAny())); + } + } + + public class TheCopyEmbeddedIconFromPackageMethod : TestBase + { + [Fact] + public async Task NoOpsIfIconFileDoesNotExist() + { + using (var packageStream = PrepareZippedImage("icon.xyz", new byte[] { 0xFF, 0xD8, 0xFF, 0x21, 0x17 })) + { + await Target.CopyEmbeddedIconFromPackageAsync(packageStream, "icon.foo", DestinationStorageMock.Object, "somepath", CancellationToken.None, "theid", "1.2.3"); + } + + DestinationStorageMock.Verify(ds => ds.SaveAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + public static IEnumerable ExtractsAndSavesIconData => + from d in ImageData + select new object[] + { + d[0], + (string)d[1] == "image/jpeg" || (string)d[1] == "image/png" ? d[1] : string.Empty + }; + + [Theory] + [MemberData(nameof(ExtractsAndSavesIconData))] + public async Task ExtractsAndSavesIcon(byte[] imageData, string expectedContentType) + { + const string iconFilename = "somefile.sxt"; + var destinationUri = new Uri("https://nuget.test/somepath"); + DestinationStorageMock + .Setup(ds => ds.ResolveUri("somepath")) + .Returns(destinationUri); + using (var packageStream = PrepareZippedImage(iconFilename, imageData)) + { + await Target.CopyEmbeddedIconFromPackageAsync(packageStream, iconFilename, DestinationStorageMock.Object, "somepath", CancellationToken.None, "theid", "1.2.3"); + } + + DestinationStorageMock.Verify( + ds => ds.SaveAsync( + It.Is(u => u == destinationUri), + It.Is(sc => SameDataAndContentType(imageData, expectedContentType, sc)), + It.IsAny()), + Times.Once); + } + + private static bool SameDataAndContentType(byte[] expectedData, string expectedContentType, StorageContent content) + { + return SameData(expectedData, content) && content.ContentType == expectedContentType; + } + + private static MemoryStream PrepareZippedImage(string imagePath, byte[] imageData) + { + var result = new MemoryStream(); + + using (var archive = new ZipArchive(result, ZipArchiveMode.Create, leaveOpen: true)) + { + var entry = archive.CreateEntry(imagePath); + using(var entryStream = entry.Open()) + { + entryStream.Write(imageData, 0, imageData.Length); + } + } + + result.Seek(0, SeekOrigin.Begin); + return result; + } + } + + public class TestBase + { + protected IconProcessor Target { get; set; } + protected Mock DestinationStorageMock { get; private set; } + protected Mock TelemetryServiceMock { get; set; } + protected Mock> LoggerMock { get; set; } + + public TestBase() + { + TelemetryServiceMock = new Mock(); + LoggerMock = new Mock>(); + + Target = new IconProcessor(TelemetryServiceMock.Object, LoggerMock.Object); + + DestinationStorageMock = new Mock(); + } + + public static IEnumerable ImageData = new[] { + new object[] { new byte[] { 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x42 }, "image/png" }, + new object[] { new byte[] { 0xFF, 0xD8, 0xFF, 0x21, 0x17 }, "image/jpeg" }, + new object[] { new byte[] { 0x47, 0x49, 0x46, 0x38, 0x37, 0x61, 0x34, 0x12 }, "image/gif" }, + new object[] { new byte[] { 0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0x45, 0x98, 0x03 }, "image/gif" }, + new object[] { new byte[] { 0x00, 0x00, 0x01, 0x00, 0x92 }, "image/x-icon" }, + new object[] { Encoding.UTF8.GetBytes(""), "image/svg+xml" } + }; + + protected static bool SameData(byte[] data, StorageContent storageContent) + { + using (var dataStream = storageContent.GetContentStream()) + using (var m = new MemoryStream()) + { + dataStream.CopyTo(m); + var submittedArray = m.ToArray(); + return data.SequenceEqual(submittedArray); + } + } + } + } +} diff --git a/tests/CatalogTests/PackageCatalogItemCreatorTests.cs b/tests/CatalogTests/PackageCatalogItemCreatorTests.cs new file mode 100644 index 000000000..29e3b5768 --- /dev/null +++ b/tests/CatalogTests/PackageCatalogItemCreatorTests.cs @@ -0,0 +1,426 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NgTests.Infrastructure; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests +{ + public class PackageCatalogItemCreatorTests + { + private const string _packageHash = "bq5DjCtCJpy9R5rsEeQlKz8qGF1Bh3wGaJKMlRwmCoKZ8WUCIFtU3JlyMOdAkSn66KCehCCAxMZFOQD4nNnH/w=="; + + private static readonly MockTelemetryService _telemetryService = new MockTelemetryService(); + + [Fact] + public void Create_WhenHttpClientIsNull_Throws() + { + HttpClient httpClient = null; + + var exception = Assert.Throws( + () => PackageCatalogItemCreator.Create( + httpClient, + Mock.Of(), + Mock.Of(), + Mock.Of())); + + Assert.Equal("httpClient", exception.ParamName); + } + + [Fact] + public void Create_WhenTelemetryServiceIsNull_Throws() + { + ITelemetryService telemetryService = null; + + var exception = Assert.Throws( + () => PackageCatalogItemCreator.Create( + Mock.Of(), + telemetryService, + Mock.Of(), + Mock.Of())); + + Assert.Equal("telemetryService", exception.ParamName); + } + + [Fact] + public void Create_WhenLoggerIsNull_Throws() + { + ILogger logger = null; + + var exception = Assert.Throws( + () => PackageCatalogItemCreator.Create( + Mock.Of(), + Mock.Of(), + logger, + Mock.Of())); + + Assert.Equal("logger", exception.ParamName); + } + + [Fact] + public void Create_WhenStorageIsNull_ReturnsInstance() + { + var creator = PackageCatalogItemCreator.Create( + Mock.Of(), + Mock.Of(), + Mock.Of(), + storage: null); + + Assert.NotNull(creator); + } + + [Fact] + public async Task CreateAsync_WhenPackageItemIsNull_Throws() + { + var creator = PackageCatalogItemCreator.Create( + Mock.Of(), + _telemetryService, + Mock.Of(), + storage: null); + + FeedPackageDetails packageItem = null; + + var exception = await Assert.ThrowsAsync( + () => creator.CreateAsync(packageItem, DateTime.UtcNow, CancellationToken.None)); + + Assert.Equal("packageItem", exception.ParamName); + } + + [Fact] + public async Task CreateAsync_WhenCancellationTokenIsCancelled_Throws() + { + var creator = PackageCatalogItemCreator.Create( + Mock.Of(), + _telemetryService, + Mock.Of(), + storage: null); + var packageItem = new FeedPackageDetails( + new Uri("https://nuget.test"), + DateTime.UtcNow, + DateTime.UtcNow, + DateTime.UtcNow, + packageId: "a", + packageNormalizedVersion: "1.0.0", + packageFullVersion: "1.0.0"); + + await Assert.ThrowsAsync( + () => creator.CreateAsync(packageItem, DateTime.UtcNow, new CancellationToken(canceled: true))); + } + + public class WhenStorageIsNotIAzureStorage + { + [Fact] + public async Task CreateAsync_WhenHttpPackageSourceReturnsPackage_ReturnsInstance() + { + using (var test = new Test(useStorage: false)) + { + var item = await test.Creator.CreateAsync(test.FeedPackageDetails, test.Timestamp, CancellationToken.None); + + AssertCorrect(item, test.FeedPackageDetails); + + Assert.Equal(1, test.TelemetryService.TrackDurationCalls.Count); + + var call = test.TelemetryService.TrackDurationCalls[0]; + + Assert.Equal("PackageDownloadSeconds", call.Name); + Assert.Equal(2, call.Properties.Count); + Assert.Equal(test.PackageId.ToLowerInvariant(), call.Properties["Id"]); + Assert.Equal(test.PackageVersion.ToLowerInvariant(), call.Properties["Version"]); + } + } + } + + public class WhenStorageIsIAzureStorage + { + [Fact] + public async Task CreateAsync_WhenAzureStorageReturnsNonexistantBlob_ReturnsInstanceFromHttpPackageSource() + { + using (var test = new Test()) + { + test.Storage.Setup(x => x.ResolveUri(test.PackageFileName)) + .Returns(test.ContentUri); + + test.Storage.Setup(x => x.GetCloudBlockBlobReferenceAsync( + It.Is(uri => uri == test.ContentUri))) + .ReturnsAsync(test.Blob.Object); + + test.Blob.Setup(x => x.ExistsAsync(It.IsAny())) + .ReturnsAsync(false); + + test.Blob.SetupGet(x => x.Uri) + .Returns(test.ContentUri); + + var item = await test.Creator.CreateAsync( + test.FeedPackageDetails, + test.Timestamp, + CancellationToken.None); + + AssertCorrect(item, test.FeedPackageDetails); + Assert.Equal(1, test.Handler.Requests.Count); + + Assert.Equal(1, test.TelemetryService.TrackMetricCalls.Count); + + var call = test.TelemetryService.TrackMetricCalls[0]; + + Assert.Equal("NonExistentBlob", call.Name); + Assert.Equal(1UL, call.Metric); + Assert.Equal(3, call.Properties.Count); + Assert.Equal(test.PackageId.ToLowerInvariant(), call.Properties["Id"]); + Assert.Equal(test.PackageVersion.ToLowerInvariant(), call.Properties["Version"]); + Assert.Equal(test.ContentUri.AbsoluteUri, call.Properties["Uri"]); + } + } + + [Fact] + public async Task CreateAsync_WhenBlobDoesNotHavePackageHash_ReturnsInstanceFromHttpPackageSource() + { + using (var test = new Test()) + { + test.Storage.Setup(x => x.ResolveUri(test.PackageFileName)) + .Returns(test.ContentUri); + + test.Storage.Setup(x => x.GetCloudBlockBlobReferenceAsync( + It.Is(uri => uri == test.ContentUri))) + .ReturnsAsync(test.Blob.Object); + + test.Blob.Setup(x => x.ExistsAsync(It.IsAny())) + .ReturnsAsync(true); + + test.Blob.Setup(x => x.FetchAttributesAsync(It.IsAny())) + .Returns(Task.FromResult(0)); + + test.Blob.SetupGet(x => x.ETag) + .Returns("0"); + + test.Blob.Setup(x => x.GetMetadataAsync(It.IsAny())) + .ReturnsAsync(new Dictionary()); + + test.Blob.SetupGet(x => x.Uri) + .Returns(test.ContentUri); + + var item = await test.Creator.CreateAsync( + test.FeedPackageDetails, + test.Timestamp, + CancellationToken.None); + + AssertCorrect(item, test.FeedPackageDetails); + Assert.Equal(1, test.Handler.Requests.Count); + + Assert.Equal(1, test.TelemetryService.TrackMetricCalls.Count); + + var call = test.TelemetryService.TrackMetricCalls[0]; + + Assert.Equal("NonExistentPackageHash", call.Name); + Assert.Equal(1UL, call.Metric); + Assert.Equal(3, call.Properties.Count); + Assert.Equal(test.PackageId.ToLowerInvariant(), call.Properties["Id"]); + Assert.Equal(test.PackageVersion.ToLowerInvariant(), call.Properties["Version"]); + Assert.Equal(test.ContentUri.AbsoluteUri, call.Properties["Uri"]); + } + } + + [Fact] + public async Task CreateAsync_WhenBlobHasPackageHash_ReturnsInstanceFromBlobStorage() + { + using (var test = new Test()) + { + test.Storage.Setup(x => x.ResolveUri(test.PackageFileName)) + .Returns(test.ContentUri); + + test.Storage.Setup(x => x.GetCloudBlockBlobReferenceAsync( + It.Is(uri => uri == test.ContentUri))) + .ReturnsAsync(test.Blob.Object); + + test.Blob.Setup(x => x.ExistsAsync(It.IsAny())) + .ReturnsAsync(true); + + test.Blob.Setup(x => x.FetchAttributesAsync(It.IsAny())) + .Returns(Task.FromResult(0)); + + test.Blob.SetupGet(x => x.ETag) + .Returns("0"); + + test.Blob.Setup(x => x.GetMetadataAsync(It.IsAny())) + .ReturnsAsync(new Dictionary() + { + { "SHA512", _packageHash } + }); + + test.Blob.Setup(x => x.GetStreamAsync(It.IsAny())) + .ReturnsAsync(TestHelper.GetStream(test.PackageFileName)); + + test.Blob.Setup(x => x.FetchAttributesAsync(It.IsAny())) + .Returns(Task.FromResult(0)); + + test.Blob.SetupGet(x => x.ETag) + .Returns("0"); + + var item = await test.Creator.CreateAsync( + test.FeedPackageDetails, + test.Timestamp, + CancellationToken.None); + + AssertCorrect(item, test.FeedPackageDetails); + Assert.Empty(test.Handler.Requests); + Assert.Empty(test.TelemetryService.TrackMetricCalls); + } + } + + [Fact] + public async Task CreateAsync_WhenBlobChangesBetweenReads_ReturnsInstanceFromHttpPackageSource() + { + using (var test = new Test()) + { + test.Storage.Setup(x => x.ResolveUri(test.PackageFileName)) + .Returns(test.ContentUri); + + test.Storage.Setup(x => x.GetCloudBlockBlobReferenceAsync( + It.Is(uri => uri == test.ContentUri))) + .ReturnsAsync(test.Blob.Object); + + test.Blob.Setup(x => x.ExistsAsync(It.IsAny())) + .ReturnsAsync(true); + + test.Blob.Setup(x => x.FetchAttributesAsync(It.IsAny())) + .Returns(Task.FromResult(0)); + + test.Blob.SetupSequence(x => x.ETag) + .Returns("0") + .Returns("1"); + + test.Blob.Setup(x => x.GetMetadataAsync(It.IsAny())) + .ReturnsAsync(new Dictionary() + { + { "SHA512", _packageHash } + }); + + test.Blob.Setup(x => x.GetStreamAsync(It.IsAny())) + .ReturnsAsync(TestHelper.GetStream(test.PackageFileName)); + + test.Blob.Setup(x => x.FetchAttributesAsync(It.IsAny())) + .Returns(Task.FromResult(0)); + + test.Blob.SetupGet(x => x.Uri) + .Returns(test.ContentUri); + + var item = await test.Creator.CreateAsync( + test.FeedPackageDetails, + test.Timestamp, + CancellationToken.None); + + AssertCorrect(item, test.FeedPackageDetails); + Assert.Single(test.Handler.Requests); + + Assert.Single(test.TelemetryService.TrackMetricCalls); + + var call = test.TelemetryService.TrackMetricCalls[0]; + + Assert.Equal("BlobModified", call.Name); + Assert.Equal(1UL, call.Metric); + Assert.Equal(3, call.Properties.Count); + Assert.Equal(test.PackageId.ToLowerInvariant(), call.Properties["Id"]); + Assert.Equal(test.PackageVersion.ToLowerInvariant(), call.Properties["Version"]); + Assert.Equal(test.ContentUri.AbsoluteUri, call.Properties["Uri"]); + } + } + } + + private static void AssertCorrect(PackageCatalogItem item, FeedPackageDetails feedPackageDetails) + { + Assert.NotNull(item); + Assert.Equal(18, item.NupkgMetadata.Entries.Count()); + Assert.NotNull(item.NupkgMetadata.Nuspec); + Assert.Equal(_packageHash, item.NupkgMetadata.PackageHash); + Assert.Equal(1871318, item.NupkgMetadata.PackageSize); + Assert.Equal(feedPackageDetails.CreatedDate, item.CreatedDate); + Assert.Equal(feedPackageDetails.LastEditedDate, item.LastEditedDate); + Assert.Equal(feedPackageDetails.PublishedDate, item.PublishedDate); + } + + private sealed class Test : IDisposable + { + private readonly HttpClient _httpClient; + private bool _isDisposed; + + internal Mock Blob { get; } + internal Uri ContentUri { get; } + internal PackageCatalogItemCreator Creator { get; } + internal FeedPackageDetails FeedPackageDetails { get; } + internal MockServerHttpClientHandler Handler { get; } + internal string PackageFileName { get; } + internal string PackageId => "Newtonsoft.Json"; + internal Uri PackageUri { get; } + internal string PackageVersion => "9.0.2-beta1"; + internal Mock Storage { get; } + internal MockTelemetryService TelemetryService { get; } + internal DateTime Timestamp { get; } + + internal Test(bool useStorage = true) + { + Handler = new MockServerHttpClientHandler(); + _httpClient = new HttpClient(Handler); + + PackageFileName = $"{PackageId.ToLowerInvariant()}.{PackageVersion.ToLowerInvariant()}.nupkg"; + Timestamp = DateTime.UtcNow; + ContentUri = new Uri($"https://nuget.test/packages/{PackageFileName}"); + PackageUri = Utilities.GetNugetCacheBustingUri(ContentUri, Timestamp.ToString("O")); + Storage = new Mock(MockBehavior.Strict); + Blob = new Mock(MockBehavior.Strict); + FeedPackageDetails = new FeedPackageDetails( + ContentUri, + Timestamp.AddHours(-3), + Timestamp.AddHours(-2), + Timestamp.AddHours(-1), + PackageId, + PackageVersion, + PackageVersion); + + var stream = TestHelper.GetStream(PackageFileName); + + Handler.SetAction( + PackageUri.PathAndQuery, + request => Task.FromResult( + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StreamContent(stream) + })); + + TelemetryService = new MockTelemetryService(); + + Creator = PackageCatalogItemCreator.Create( + _httpClient, + TelemetryService, + Mock.Of(), + useStorage ? Storage.Object : null); + } + + public void Dispose() + { + if (!_isDisposed) + { + Blob.VerifyAll(); + Storage.VerifyAll(); + + Handler.Dispose(); + _httpClient.Dispose(); + + GC.SuppressFinalize(this); + + _isDisposed = true; + } + } + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/PackageCatalogItemTests.cs b/tests/CatalogTests/PackageCatalogItemTests.cs new file mode 100644 index 000000000..627232c68 --- /dev/null +++ b/tests/CatalogTests/PackageCatalogItemTests.cs @@ -0,0 +1,191 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Linq; +using Moq; +using NuGet.Services.Metadata.Catalog; +using VDS.RDF; +using Xunit; + +namespace CatalogTests.Helpers +{ + public class PackageCatalogItemTests + { + [Theory] + [InlineData("Newtonsoft.Json.9.0.2-beta1")] + [InlineData("TestPackage.SemVer2.1.0.0-alpha.1")] + [InlineData("DependencyMissingId.0.1.0")] // One dependency missing an ID attribute + [InlineData("EmptyDependenciesElement.0.1.0")] // A element with no children + [InlineData("EmptyDependencyId.0.1.0")] // One dependency with an empty string ID + [InlineData("EmptyDependencyIdWithGroups.0.1.0")] // Using dependency groups, one dependency with an empty string ID + [InlineData("OneValidDependencyOneEmptyId.0.1.0")] // One valid dependency and one with empty string ID + [InlineData("OneValidDependencyOneEmptyIdWithGroups.0.1.0")] // Using dependency groups, one valid dependency and one with empty string ID + [InlineData("WhitespaceDependencyId.0.1.0")] // One dependency with an ID only containing whitespace + [InlineData("EmptyDependencyVersionRange.0.1.0")] // A dependency with a version range that is an empty string + [InlineData("InvalidDependencyVersionRange.0.1.0")] //A dependency with a version range that is invalid + [InlineData("MissingDependencyVersionRange.0.1.0")] // A dependency with no version range attribute + [InlineData("WhitespaceDependencyVersionRange.0.1.0")] // A dependency with a version range that is whitespace + [InlineData("PackageTypeCollapseDuplicate")] + [InlineData("PackageTypeMultiple")] + [InlineData("PackageTypeMultipleTypesNodes")] + [InlineData("PackageTypeSingle")] + [InlineData("PackageTypeSingleWithVersion")] + [InlineData("PackageTypeSameTypeDifferentCase")] + [InlineData("PackageTypeSameTypeDifferentVersionType")] + [InlineData("PackageTypeSameTypeTwoVersion")] + [InlineData("PackageTypeWhiteSpace")] + [InlineData("PackageTypeWhiteSpaceVersion")] + public void CreateContent_ProducesExpectedJson(string packageName) + { + // Arrange + var catalogItem = CreateCatalogItem(packageName); + var catalogContext = new CatalogContext(); + + // Act + var content = catalogItem.CreateContent(catalogContext); + + // Assert + var expected = File.ReadAllText(Path.Combine("TestData", $"{packageName}.json")); + string actual; + using (var reader = new StreamReader(content.GetContentStream())) + { + actual = reader.ReadToEnd(); + } + + Assert.Equal("no-store", content.CacheControl); + Assert.Equal("application/json", content.ContentType); + Assert.Equal(expected, actual); + } + + [Theory] + [InlineData("Newtonsoft.Json.9.0.2-beta1", "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json")] + [InlineData("TestPackage.SemVer2.1.0.0-alpha.1", "http://example/data/2017.01.04.08.15.00/testpackage.semver2.1.0.0-alpha.1.json")] + public void CreateContent_HasExpectedItemAddress(string packageName, string expected) + { + // Arrange + var catalogItem = CreateCatalogItem(packageName); + var catalogContext = new CatalogContext(); + catalogItem.CreateContent(catalogContext); + + // Act + var itemAddress = catalogItem.GetItemAddress(); + + // Assert + Assert.Equal(new Uri(expected), itemAddress); + } + + [Theory] + [InlineData("Newtonsoft.Json.9.0.2-beta1")] + [InlineData("TestPackage.SemVer2.1.0.0-alpha.1")] + public void CreateContent_HasExpectedItemType(string packageName) + { + // Arrange + var catalogItem = CreateCatalogItem(packageName); + var catalogContext = new CatalogContext(); + catalogItem.CreateContent(catalogContext); + + // Act + var itemAddress = catalogItem.GetItemType(); + + // Assert + Assert.Equal(new Uri("http://schema.nuget.org/schema#PackageDetails"), itemAddress); + } + + [Theory] + [InlineData("Newtonsoft.Json.9.0.2-beta1", "Newtonsoft.Json", "9.0.2-beta1")] + [InlineData("TestPackage.SemVer2.1.0.0-alpha.1", "TestPackage.SemVer2", "1.0.0-alpha.1+githash")] + public void CreateContent_HasExpectedPageContent(string packageName, string id, string version) + { + // Arrange + var catalogItem = CreateCatalogItem(packageName); + var catalogContext = new CatalogContext(); + catalogItem.CreateContent(catalogContext); + + // Act + var pageContent = catalogItem.CreatePageContent(catalogContext); + + // Assert + var triples = pageContent + .Triples + .Cast() + .OrderBy(x => x.Subject) + .ThenBy(x => x.Predicate) + .ToList(); + Assert.Equal(2, triples.Count); + Assert.Equal(Schema.Predicates.Id.ToString(), triples[0].Predicate.ToString()); + Assert.Equal(id, triples[0].Object.ToString()); + Assert.Equal(Schema.Predicates.Version.ToString(), triples[1].Predicate.ToString()); + Assert.Equal(version, triples[1].Object.ToString()); + } + + [Fact] + public void CreateContent_ThrowsIfMultipleDeprecationTriples() + { + var packageDetails = Schema.DataTypes.PackageDetails; + var catalogItemMock = new Mock(null, null, null, null, null, null, null, null) + { + CallBase = true + }; + + var context = new CatalogContext(); + + catalogItemMock + .Setup(x => x.GetItemType()) + .Returns(packageDetails); + + var graph = new Graph(); + var subject = graph.CreateBlankNode(); + graph.Assert( + subject, + graph.CreateUriNode(Schema.Predicates.Type), + graph.CreateUriNode(packageDetails)); + + graph.Assert( + subject, + graph.CreateUriNode(Schema.Predicates.Deprecation), + graph.CreateLiteralNode("deprecation1")); + + graph.Assert( + subject, + graph.CreateUriNode(Schema.Predicates.Deprecation), + graph.CreateLiteralNode("deprecation2")); + + catalogItemMock + .Setup(x => x.CreateContentGraph(context)) + .Returns(graph); + + Assert.Throws( + () => catalogItemMock.Object.CreateContent(context)); + } + + private static CatalogItem CreateCatalogItem(string packageName) + { + var path = Path.GetFullPath(Path.Combine("TestData", $"{packageName}.nupkg")); + + using (var packageStream = TestHelper.GetStream($"{packageName}.nupkg")) + { + var createdDate = new DateTime(2017, 1, 1, 8, 15, 0, DateTimeKind.Utc); + var lastEditedDate = new DateTime(2017, 1, 2, 8, 15, 0, DateTimeKind.Utc); + var publishedDate = new DateTime(2017, 1, 3, 8, 15, 0, DateTimeKind.Utc); + + var baseAddress = new Uri("http://example/catalog"); + var timestamp = new DateTime(2017, 1, 4, 8, 15, 0, DateTimeKind.Utc); + var commitId = new Guid("4AEE0EF4-A039-4460-BD5F-98F944E33289"); + + var catalogItem = Utils.CreateCatalogItem( + path, + packageStream, + createdDate, + lastEditedDate, + publishedDate); + catalogItem.TimeStamp = timestamp; + catalogItem.CommitId = commitId; + catalogItem.BaseAddress = baseAddress; + + return catalogItem; + } + } + } +} diff --git a/tests/CatalogTests/PackageDeprecationItemTests.cs b/tests/CatalogTests/PackageDeprecationItemTests.cs new file mode 100644 index 000000000..36c071858 --- /dev/null +++ b/tests/CatalogTests/PackageDeprecationItemTests.cs @@ -0,0 +1,54 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Services.Metadata.Catalog; +using Xunit; + +namespace CatalogTests +{ + public class PackageDeprecationItemTests + { + public class TheConstructor + { + [Fact] + public void ThrowsIfNullReasons() + { + Assert.Throws(() => new PackageDeprecationItem(null, null, null, null)); + } + + [Fact] + public void ThrowsIfEmptyReasons() + { + Assert.Throws(() => new PackageDeprecationItem(new string[0], null, null, null)); + } + + [Fact] + public void ThrowsIfVersionRangeProvidedWithoutId() + { + Assert.Throws(() => new PackageDeprecationItem(new[] { "first", "second" }, null, null, "howdy")); + } + + [Fact] + public void ThrowsIfIdProvidedWithoutVersionRange() + { + Assert.Throws(() => new PackageDeprecationItem(new[] { "first", "second" }, null, "howdy", null)); + } + + [Fact] + public void SetsExpectedValues() + { + var reasons = new[] { "first", "second" }; + var message = "message"; + var id = "theId"; + var versionRange = "homeOnTheRange"; + + var deprecation = new PackageDeprecationItem(reasons, message, id, versionRange); + Assert.Equal(reasons, deprecation.Reasons); + Assert.Equal(message, deprecation.Message); + Assert.Equal(id, deprecation.AlternatePackageId); + Assert.Equal(versionRange, deprecation.AlternatePackageRange); + } + } + } +} diff --git a/tests/CatalogTests/PackageEntryTests.cs b/tests/CatalogTests/PackageEntryTests.cs new file mode 100644 index 000000000..76350ca00 --- /dev/null +++ b/tests/CatalogTests/PackageEntryTests.cs @@ -0,0 +1,97 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.IO.Compression; +using Newtonsoft.Json; +using NuGet.Services.Metadata.Catalog; +using Xunit; + +namespace CatalogTests +{ + public class PackageEntryTests + { + [Fact] + public void DefaultConstructor_InitializesDefaultValues() + { + var packageEntry = new PackageEntry(); + + Assert.Null(packageEntry.FullName); + Assert.Null(packageEntry.Name); + Assert.Equal(0, packageEntry.CompressedLength); + Assert.Equal(0, packageEntry.Length); + } + + [Fact] + public void Constructor_WhenZipArchiveEntryIsNull_Throws() + { + var exception = Assert.Throws(() => new PackageEntry(zipArchiveEntry: null)); + + Assert.Equal("zipArchiveEntry", exception.ParamName); + } + + [Fact] + public void Constructor_WithValidArguments_InitializesInstance() + { + using (var zipArchive = CreateZipArchive()) + { + var zipArchiveEntry = zipArchive.Entries[0]; + + var packageEntry = new PackageEntry(zipArchiveEntry); + + Assert.Equal(zipArchiveEntry.FullName, packageEntry.FullName); + Assert.Equal(zipArchiveEntry.Name, packageEntry.Name); + Assert.Equal(zipArchiveEntry.CompressedLength, packageEntry.CompressedLength); + Assert.Equal(zipArchiveEntry.Length, packageEntry.Length); + } + } + + [Fact] + public void JsonSerialization_ReturnsCorrectJson() + { + var packageEntry = new PackageEntry() + { + FullName = "a/b", + Name = "b", + CompressedLength = 2, + Length = 1 + }; + + var json = JsonConvert.SerializeObject(packageEntry); + + Assert.Equal("{\"fullName\":\"a/b\",\"name\":\"b\",\"length\":1,\"compressedLength\":2}", json); + } + + [Fact] + public void JsonDeserialization_ReturnsCorrectObject() + { + var json = "{\"fullName\":\"a/b\",\"name\":\"b\",\"length\":1,\"compressedLength\":2}"; + + var packageEntry = JsonConvert.DeserializeObject(json); + + Assert.Equal("a/b", packageEntry.FullName); + Assert.Equal("b", packageEntry.Name); + Assert.Equal(1, packageEntry.Length); + Assert.Equal(2, packageEntry.CompressedLength); + } + + private static ZipArchive CreateZipArchive() + { + var archiveStream = new MemoryStream(); + + using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Create, leaveOpen: true)) + { + var entry = archive.CreateEntry("a/b.c", CompressionLevel.Optimal); + + using (var entryStream = entry.Open()) + using (var writer = new StreamWriter(entryStream)) + { + writer.Write("peach"); + } + } + + return new ZipArchive(archiveStream, ZipArchiveMode.Read); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Persistence/AzureCloudBlockBlobTests.cs b/tests/CatalogTests/Persistence/AzureCloudBlockBlobTests.cs new file mode 100644 index 000000000..4fba0af1d --- /dev/null +++ b/tests/CatalogTests/Persistence/AzureCloudBlockBlobTests.cs @@ -0,0 +1,113 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Moq; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests.Persistence +{ + public class AzureCloudBlockBlobTests + { + private static readonly Uri _uri = new Uri("https://nuget.test/blob"); + + private readonly Mock _underlyingBlob; + + public AzureCloudBlockBlobTests() + { + _underlyingBlob = new Mock(MockBehavior.Strict, _uri); + } + + [Fact] + public void Constructor_WhenBlobIsNull_Throws() + { + var exception = Assert.Throws(() => new AzureCloudBlockBlob(blob: null)); + + Assert.Equal("blob", exception.ParamName); + } + + [Fact] + public async Task ExistsAsync_CallsUnderlyingMethod() + { + _underlyingBlob.Setup(x => x.ExistsAsync()) + .ReturnsAsync(true); + + var blob = new AzureCloudBlockBlob(_underlyingBlob.Object); + + Assert.True(await blob.ExistsAsync(CancellationToken.None)); + + _underlyingBlob.VerifyAll(); + } + + [Fact] + public async Task FetchAttributesAsync_CallsUnderlyingMethod() + { + _underlyingBlob.Setup(x => x.FetchAttributesAsync(It.IsAny())) + .Returns(Task.FromResult(0)); + + var blob = new AzureCloudBlockBlob(_underlyingBlob.Object); + + await blob.FetchAttributesAsync(CancellationToken.None); + + _underlyingBlob.VerifyAll(); + } + + // CloudBlockBlob.Metadata is non-virtual, which blocks more thorough testing. + [Fact] + public async Task GetMetadataAsync_ReturnsReadOnlyDictionary() + { + var blob = new AzureCloudBlockBlob(_underlyingBlob.Object); + + var actualMetadata = await blob.GetMetadataAsync(CancellationToken.None); + + Assert.IsAssignableFrom>(actualMetadata); + + _underlyingBlob.VerifyAll(); + } + + [Fact] + public async Task GetStreamAsync_CallsUnderlyingMethod() + { + var expectedStream = new MemoryStream(); + + _underlyingBlob.Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(expectedStream); + + var blob = new AzureCloudBlockBlob(_underlyingBlob.Object); + + var actualStream = await blob.GetStreamAsync(CancellationToken.None); + + Assert.Same(expectedStream, actualStream); + + _underlyingBlob.VerifyAll(); + } + + [Fact] + public async Task SetPropertiesAsync_CallsUnderlyingMethod() + { + // Arrange + var blob = new AzureCloudBlockBlob(_underlyingBlob.Object); + + var accessCondition = AccessCondition.GenerateEmptyCondition(); + var options = new BlobRequestOptions(); + var operationContext = new OperationContext(); + + _underlyingBlob + .Setup(b => b.SetPropertiesAsync(accessCondition, options, operationContext)) + .Returns(Task.CompletedTask); + + // Act + await blob.SetPropertiesAsync(accessCondition, options, operationContext); + + // Assert + _underlyingBlob.VerifyAll(); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Persistence/FileStorageTests.cs b/tests/CatalogTests/Persistence/FileStorageTests.cs new file mode 100644 index 000000000..7c92be8f6 --- /dev/null +++ b/tests/CatalogTests/Persistence/FileStorageTests.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests.Persistence +{ + public class FileStorageTests + { + private readonly FileStorage _storage; + + public FileStorageTests() + { + _storage = CreateNewFileStorage(); + } + + [Fact] + public async Task CopyAsync_Always_Throws() + { + var destinationStorage = CreateNewFileStorage(); + var sourceFileUri = _storage.ResolveUri("a"); + var destinationFileUri = destinationStorage.ResolveUri("a"); + + await Assert.ThrowsAsync( + () => _storage.CopyAsync( + sourceFileUri, + destinationStorage, + destinationFileUri, + destinationProperties: null, + cancellationToken: CancellationToken.None)); + } + + private static FileStorage CreateNewFileStorage() + { + return new FileStorage(Path.GetTempPath(), Guid.NewGuid().ToString("N"), verbose: false); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/Persistence/OptimisticConcurrencyControlTokenTests.cs b/tests/CatalogTests/Persistence/OptimisticConcurrencyControlTokenTests.cs new file mode 100644 index 000000000..8361e0b30 --- /dev/null +++ b/tests/CatalogTests/Persistence/OptimisticConcurrencyControlTokenTests.cs @@ -0,0 +1,59 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace CatalogTests.Persistence +{ + public class OptimisticConcurrencyControlTokenTests + { + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("a")] + public void WhenTokensAreEqual_EqualityTestsSucceed(string innerToken) + { + var token1 = new OptimisticConcurrencyControlToken(innerToken); + var token2 = new OptimisticConcurrencyControlToken(innerToken); + + Assert.True(token1.Equals(token2)); + Assert.True(token1 == token2); + Assert.False(token1 != token2); + } + + [Theory] + [InlineData(null, "")] + [InlineData("", "a")] + [InlineData("a", "b")] + public void WhenTokensAreNotEqual_EqualityTestsFail(string innerToken1, string innerToken2) + { + var token1 = new OptimisticConcurrencyControlToken(innerToken1); + var token2 = new OptimisticConcurrencyControlToken(innerToken2); + + Assert.False(token1.Equals(token2)); + Assert.False(token1 == token2); + Assert.True(token1 != token2); + } + + [Fact] + public void Null_EqualsNullToken() + { + var token1 = OptimisticConcurrencyControlToken.Null; + var token2 = OptimisticConcurrencyControlToken.Null; + + Assert.True(token1.Equals(token2)); + Assert.True(token1 == token2); + Assert.False(token1 != token2); + } + + [Fact] + public void GetHashCode_EqualsInnerTokenGetHashCode() + { + var innerToken = "abc"; + var token = new OptimisticConcurrencyControlToken(innerToken); + + Assert.Equal(innerToken.GetHashCode(), token.GetHashCode()); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/PipelineTest.cs b/tests/CatalogTests/PipelineTest.cs new file mode 100644 index 000000000..701cf779e --- /dev/null +++ b/tests/CatalogTests/PipelineTest.cs @@ -0,0 +1,30 @@ +using NuGet.Services.Metadata.Catalog.Pipeline; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CatalogTests +{ + static class PipelineTest + { + public static void Test0() + { + PackagePipeline pipeline = PackagePipelineFactory.Create2(); + + Stream stream = new FileStream(@"C:\data\ema\nupkgs\BasicAppPackage.1.0.0.nupkg", FileMode.Open); + + PipelinePackage package = new PipelinePackage(stream); + + PackagePipelineContext context = new PackagePipelineContext(new Uri("http://azure.com/siena/package")); + + pipeline.Execute(package, context); + + PackageMetadataBase result = context.Result; + + Console.WriteLine(result.ToContent()); + } + } +} diff --git a/tests/CatalogTests/Properties/AssemblyInfo.cs b/tests/CatalogTests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..4ac107f3d --- /dev/null +++ b/tests/CatalogTests/Properties/AssemblyInfo.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System.Reflection; +using System.Runtime.InteropServices; +using Xunit; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("CatalogTests")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("CatalogTests")] +[assembly: AssemblyCopyright("Copyright © 2014")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("027675ae-93ec-4da5-89da-bc8db94359fc")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] + +// All classes that use RegistrationMakerCatalogItem.PackagePathProvider are not thread safe and cannot be tested in parallel. +// Disabling test parallelization to prevent random unexpected failures due to race conditions. +// https://github.com/NuGet/Engineering/issues/2410 +[assembly: CollectionBehavior(DisableTestParallelization = true)] \ No newline at end of file diff --git a/tests/CatalogTests/ReadOnlyGraphTests.cs b/tests/CatalogTests/ReadOnlyGraphTests.cs new file mode 100644 index 000000000..e97dcbade --- /dev/null +++ b/tests/CatalogTests/ReadOnlyGraphTests.cs @@ -0,0 +1,211 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using NuGet.Services.Metadata.Catalog; +using VDS.RDF; +using Xunit; + +namespace CatalogTests +{ + public class ReadOnlyGraphTests + { + private const string PackageId = "Newtonsoft.Json"; + private const string PackageVersion = "9.0.1"; + private const string ReadOnlyMessage = "This RDF graph cannot be modified."; + private static readonly Uri PackageUri = new Uri("http://example/newtonsoft.json/9.0.1.json"); + + [Fact] + public void MutationsOfTheOriginalGraphDoNotEffectTheReadOnlyGraph() + { + // Arrange + var mutableGraph = GetIdMutableGraph(); + var readOnlyGraph = new ReadOnlyGraph(mutableGraph); + + // Act + mutableGraph.Assert( + mutableGraph.CreateUriNode(PackageUri), + mutableGraph.CreateUriNode(Schema.Predicates.Version), + mutableGraph.CreateLiteralNode(PackageVersion)); + + // Assert + var versionTriples = readOnlyGraph.GetTriplesWithSubjectPredicate( + readOnlyGraph.CreateUriNode(PackageUri), + readOnlyGraph.CreateUriNode(Schema.Predicates.Version)); + Assert.Empty(versionTriples); + + VerifyIdTriple(readOnlyGraph); + Assert.Equal(1, readOnlyGraph.Triples.Count); + } + + [Fact] + public void RejectsAssertOfSubjectPredicateObject() + { + // Arrange + var readOnlyGraph = new ReadOnlyGraph(GetIdMutableGraph()); + + // Act & Assert + var exception = Assert.Throws(() => readOnlyGraph.Assert( + readOnlyGraph.CreateUriNode(PackageUri), + readOnlyGraph.CreateUriNode(Schema.Predicates.Version), + readOnlyGraph.CreateLiteralNode(PackageId))); + Assert.Equal(ReadOnlyMessage, exception.Message); + + VerifyIdTriple(readOnlyGraph); + Assert.Equal(1, readOnlyGraph.Triples.Count); + } + + [Fact] + public void RejectsAssertOfSingleTriple() + { + // Arrange + var readOnlyGraph = new ReadOnlyGraph(GetIdMutableGraph()); + + // Act & Assert + var exception = Assert.Throws(() => readOnlyGraph.Assert(new Triple( + readOnlyGraph.CreateUriNode(PackageUri), + readOnlyGraph.CreateUriNode(Schema.Predicates.Version), + readOnlyGraph.CreateLiteralNode(PackageId)))); + Assert.Equal(ReadOnlyMessage, exception.Message); + + VerifyIdTriple(readOnlyGraph); + Assert.Equal(1, readOnlyGraph.Triples.Count); + } + + [Fact] + public void RejectsAssertOfTripleEnumerable() + { + // Arrange + var readOnlyGraph = new ReadOnlyGraph(GetIdMutableGraph()); + + // Act & Assert + var exception = Assert.Throws(() => readOnlyGraph.Assert(new[] + { + new Triple( + readOnlyGraph.CreateUriNode(PackageUri), + readOnlyGraph.CreateUriNode(Schema.Predicates.Version), + readOnlyGraph.CreateLiteralNode(PackageVersion)) + })); + Assert.Equal(ReadOnlyMessage, exception.Message); + + VerifyIdTriple(readOnlyGraph); + Assert.Equal(1, readOnlyGraph.Triples.Count); + } + + [Fact] + public void RejectsMergeWhenKeepOriginalGraphUriIsNotSpecified() + { + // Arrange + var readOnlyGraph = new ReadOnlyGraph(GetIdMutableGraph()); + var otherGraph = GetVersionMutableGraph(); + + // Act & Assert + var exception = Assert.Throws(() => readOnlyGraph.Merge(otherGraph)); + Assert.Equal(ReadOnlyMessage, exception.Message); + + VerifyIdTriple(readOnlyGraph); + Assert.Equal(1, readOnlyGraph.Triples.Count); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void RejectsMergeWhenKeepOriginalGraphUriIsSpecified(bool keepOriginalGraphUri) + { + // Arrange + var readOnlyGraph = new ReadOnlyGraph(GetIdMutableGraph()); + var otherGraph = GetVersionMutableGraph(); + + // Act & Assert + var exception = Assert.Throws(() => readOnlyGraph.Merge(otherGraph, keepOriginalGraphUri)); + Assert.Equal(ReadOnlyMessage, exception.Message); + + VerifyIdTriple(readOnlyGraph); + Assert.Equal(1, readOnlyGraph.Triples.Count); + } + + [Fact] + public void RejectsRetractOfSubjectPredicateObject() + { + // Arrange + var readOnlyGraph = new ReadOnlyGraph(GetIdMutableGraph()); + + // Act & Assert + var exception = Assert.Throws(() => readOnlyGraph.Retract( + readOnlyGraph.CreateUriNode(PackageUri), + readOnlyGraph.CreateUriNode(Schema.Predicates.Id), + readOnlyGraph.CreateLiteralNode(PackageId))); + Assert.Equal(ReadOnlyMessage, exception.Message); + + VerifyIdTriple(readOnlyGraph); + Assert.Equal(1, readOnlyGraph.Triples.Count); + } + + [Fact] + public void RejectsRetractOfSingleTriple() + { + // Arrange + var readOnlyGraph = new ReadOnlyGraph(GetIdMutableGraph()); + + // Act & Assert + var exception = Assert.Throws(() => readOnlyGraph.Retract(new Triple( + readOnlyGraph.CreateUriNode(PackageUri), + readOnlyGraph.CreateUriNode(Schema.Predicates.Id), + readOnlyGraph.CreateLiteralNode(PackageId)))); + Assert.Equal(ReadOnlyMessage, exception.Message); + + VerifyIdTriple(readOnlyGraph); + Assert.Equal(1, readOnlyGraph.Triples.Count); + } + + [Fact] + public void RejectsRetractOfTripleEnumerable() + { + // Arrange + var readOnlyGraph = new ReadOnlyGraph(GetIdMutableGraph()); + + // Act & Assert + var exception = Assert.Throws(() => readOnlyGraph.Retract(new[] + { + new Triple( + readOnlyGraph.CreateUriNode(PackageUri), + readOnlyGraph.CreateUriNode(Schema.Predicates.Id), + readOnlyGraph.CreateLiteralNode(PackageId)) + })); + Assert.Equal(ReadOnlyMessage, exception.Message); + + VerifyIdTriple(readOnlyGraph); + Assert.Equal(1, readOnlyGraph.Triples.Count); + } + + private static Graph GetIdMutableGraph() + { + var mutableGraph = new Graph(); + mutableGraph.Assert( + mutableGraph.CreateUriNode(PackageUri), + mutableGraph.CreateUriNode(Schema.Predicates.Id), + mutableGraph.CreateLiteralNode(PackageId)); + return mutableGraph; + } + + private static Graph GetVersionMutableGraph() + { + var mutableGraph = new Graph(); + mutableGraph.Assert( + mutableGraph.CreateUriNode(PackageUri), + mutableGraph.CreateUriNode(Schema.Predicates.Version), + mutableGraph.CreateLiteralNode(PackageVersion)); + return mutableGraph; + } + + private static void VerifyIdTriple(ReadOnlyGraph readOnlyGraph) + { + var idTriples = readOnlyGraph.GetTriplesWithSubjectPredicate( + readOnlyGraph.CreateUriNode(PackageUri), + readOnlyGraph.CreateUriNode(Schema.Predicates.Id)); + Assert.Single(idTriples); + Assert.Equal(PackageId, ((LiteralNode)idTriples.First().Object).Value); + } + } +} diff --git a/tests/CatalogTests/Registration/FlatContainerPackagePathProviderTests.cs b/tests/CatalogTests/Registration/FlatContainerPackagePathProviderTests.cs new file mode 100644 index 000000000..4b8766a58 --- /dev/null +++ b/tests/CatalogTests/Registration/FlatContainerPackagePathProviderTests.cs @@ -0,0 +1,53 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Metadata.Catalog; +using Xunit; + +namespace CatalogTests.Registration +{ + public class FlatContainerPackagePathProviderTests + { + [Theory] + [InlineData("Newtonsoft.Json", "9.0.1", "newtonsoft.json/9.0.1/newtonsoft.json.9.0.1")] + [InlineData("Newtonsoft.Json", "9.0.1+githash", "newtonsoft.json/9.0.1/newtonsoft.json.9.0.1")] + [InlineData("Newtonsoft.Json", "9.0.1-a", "newtonsoft.json/9.0.1-a/newtonsoft.json.9.0.1-a")] + [InlineData("Newtonsoft.Json", "9.0.1-a+githash", "newtonsoft.json/9.0.1-a/newtonsoft.json.9.0.1-a")] + [InlineData("Newtonsoft.Json", "9.0.1-a.1", "newtonsoft.json/9.0.1-a.1/newtonsoft.json.9.0.1-a.1")] + [InlineData("Newtonsoft.Json", "9.0.1-a.1+githash", "newtonsoft.json/9.0.1-a.1/newtonsoft.json.9.0.1-a.1")] + public void GetPackagePath_UsesNormalizedVersion(string id, string version, string expected) + { + var flatContainerName = "v3-flatcontainer"; + + // Arrange + var provider = new FlatContainerPackagePathProvider(flatContainerName); + + // Act + var path = provider.GetPackagePath(id, version); + + // Assert + Assert.Equal($"{flatContainerName}/{expected}.nupkg", path); + } + + [Theory] + [InlineData("Newtonsoft.Json", "9.0.1", "newtonsoft.json/9.0.1")] + [InlineData("Newtonsoft.Json", "9.0.1+githash", "newtonsoft.json/9.0.1")] + [InlineData("Newtonsoft.Json", "9.0.1-a", "newtonsoft.json/9.0.1-a")] + [InlineData("Newtonsoft.Json", "9.0.1-a+githash", "newtonsoft.json/9.0.1-a")] + [InlineData("Newtonsoft.Json", "9.0.1-a.1", "newtonsoft.json/9.0.1-a.1")] + [InlineData("Newtonsoft.Json", "9.0.1-a.1+githash", "newtonsoft.json/9.0.1-a.1")] + public void GetIconPath_UsesNormalizedVersion(string id, string version, string expected) + { + var flatContainerName = "v3-flatcontainer"; + + // Arrange + var provider = new FlatContainerPackagePathProvider(flatContainerName); + + // Act + var path = provider.GetIconPath(id, version); + + // Assert + Assert.Equal($"{flatContainerName}/{expected}/icon", path); + } + } +} diff --git a/tests/CatalogTests/RetryWithExponentialBackoffTests.cs b/tests/CatalogTests/RetryWithExponentialBackoffTests.cs new file mode 100644 index 000000000..f2598c168 --- /dev/null +++ b/tests/CatalogTests/RetryWithExponentialBackoffTests.cs @@ -0,0 +1,118 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using NuGet.Services.Metadata.Catalog; +using Xunit; + +namespace CatalogTests +{ + public class RetryWithExponentialBackoffTests + { + public class TheIsTransientErrorMethod + { + [Fact] + public void ReturnsFalseIfNotCorrectExceptionType() + { + var e = new Exception(); + var result = RetryWithExponentialBackoff.IsTransientError(e, null); + Assert.False(result); + } + + public static IEnumerable TransientExceptions => new Exception[] + { + new HttpRequestException(), + new OperationCanceledException() + }; + + public static IEnumerable ReturnsTrueIfResponseNull_Data + { + get + { + foreach (var exception in TransientExceptions) + { + yield return new object[] { exception }; + } + } + } + + [Theory] + [MemberData(nameof(ReturnsTrueIfResponseNull_Data))] + public void ReturnsTrueIfResponseNull(Exception e) + { + var result = RetryWithExponentialBackoff.IsTransientError(e, null); + Assert.True(result); + } + + public static IEnumerable NonTransientStatusCodes => new[] + { + HttpStatusCode.Accepted, + HttpStatusCode.BadRequest, + HttpStatusCode.Conflict, + HttpStatusCode.NotFound, + HttpStatusCode.OK, + HttpStatusCode.Unauthorized, + HttpStatusCode.Forbidden, + HttpStatusCode.NotImplemented, + HttpStatusCode.HttpVersionNotSupported + }; + + public static IEnumerable ReturnsFalseIfResponseStatusBelow500OrWhitelisted_Data + { + get + { + foreach (var exception in TransientExceptions) + { + foreach (var status in NonTransientStatusCodes) + { + yield return new object[] { exception, status }; + } + } + } + } + + [Theory] + [MemberData(nameof(ReturnsFalseIfResponseStatusBelow500OrWhitelisted_Data))] + public void ReturnsFalseIfResponseStatusBelow500OrWhitelisted(Exception e, HttpStatusCode status) + { + var response = new HttpResponseMessage(status); + var result = RetryWithExponentialBackoff.IsTransientError(e, response); + Assert.False(result); + } + + public static IEnumerable TransientStatusCodes => new[] + { + HttpStatusCode.InternalServerError, + HttpStatusCode.BadGateway, + HttpStatusCode.ServiceUnavailable, + HttpStatusCode.GatewayTimeout + }; + + public static IEnumerable ReturnsTrueIfResponseStatusAbove500_Data + { + get + { + foreach (var exception in TransientExceptions) + { + foreach (var status in TransientStatusCodes) + { + yield return new object[] { exception, status }; + } + } + } + } + + [Theory] + [MemberData(nameof(ReturnsTrueIfResponseStatusAbove500_Data))] + public void ReturnsTrueIfResponseStatusAbove500(Exception e, HttpStatusCode status) + { + var response = new HttpResponseMessage(status); + var result = RetryWithExponentialBackoff.IsTransientError(e, response); + Assert.True(result); + } + } + } +} diff --git a/tests/CatalogTests/StringInternerTests.cs b/tests/CatalogTests/StringInternerTests.cs new file mode 100644 index 000000000..5ac016aea --- /dev/null +++ b/tests/CatalogTests/StringInternerTests.cs @@ -0,0 +1,46 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Text; +using NuGet.Services.Metadata.Catalog; +using Xunit; + +namespace CatalogTests +{ + public class StringInternerTests + { + [Fact] + public void DifferentInstancesBecomeTheSame() + { + // Arrange + var aBefore = new StringBuilder("a").Append("b").ToString(); + var bBefore = new StringBuilder("a").Append("b").ToString(); + var interner = new StringInterner(); + + // Act + var aAfter = interner.Intern(aBefore); + var bAfter = interner.Intern(aBefore); + + // Assert + Assert.NotSame(aBefore, bBefore); + Assert.Same(aAfter, bAfter); + } + + [Fact] + public void SameInstancesStayTheSame() + { + // Arrange + var aBefore = "a"; + var bBefore = aBefore; + var interner = new StringInterner(); + + // Act + var aAfter = interner.Intern(aBefore); + var bAfter = interner.Intern(aBefore); + + // Assert + Assert.Same(aBefore, bBefore); + Assert.Same(aAfter, bAfter); + } + } +} diff --git a/tests/CatalogTests/TelemetryHandlerTests.cs b/tests/CatalogTests/TelemetryHandlerTests.cs new file mode 100644 index 000000000..1e7c5e523 --- /dev/null +++ b/tests/CatalogTests/TelemetryHandlerTests.cs @@ -0,0 +1,174 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ApplicationInsights; +using Moq; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog; +using Xunit; + +namespace CatalogTests +{ + public class TelemetryHandlerTests + { + private readonly Mock _telemetryService; + private Func> _sendAsync; + private readonly FuncHttpMessageHandler _innerHandler; + private readonly HttpRequestMessage _request; + private readonly TelemetryHandler _target; + private readonly HttpClient _httpClient; + private IDictionary _properties; + + public TelemetryHandlerTests() + { + _telemetryService = new Mock(new TelemetryClientWrapper(new TelemetryClient()), new Dictionary()); + _telemetryService.Setup(x => x.TrackDuration(TelemetryConstants.HttpHeaderDurationSeconds, It.IsAny>())) + .Callback((string name, IDictionary properties) => { _properties = properties; }).CallBase(); + + _sendAsync = () => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("Hello, world!"), + }); + _innerHandler = new FuncHttpMessageHandler(() => _sendAsync); + _request = new HttpRequestMessage(HttpMethod.Get, "http://example/robots.txt"); + _target = new TelemetryHandler(_telemetryService.Object, _innerHandler); + _httpClient = new HttpClient(_target); + } + + [Fact] + public async Task EmitsTelemetryOnException() + { + // Arrange + var expected = new HttpRequestException("Bad!"); + _sendAsync = () => throw expected; + + // Act & Assert + var actual = await Assert.ThrowsAsync(() => _httpClient.SendAsync(_request)); + Assert.Same(expected, actual); + + _telemetryService.Verify(x => x.TrackDuration(It.IsAny(), It.IsAny>()), Times.Once); + + Assert.Equal(2, _properties.Count); + Assert.Equal(_request.Method.ToString(), _properties[TelemetryConstants.Method]); + Assert.Equal(_request.RequestUri.AbsoluteUri, _properties[TelemetryConstants.Uri]); + } + + [Fact] + public async Task EmitsTelemetryOnFailedRequest() + { + // Arrange + var expected = new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("Bad!"), + }; + _sendAsync = () => Task.FromResult(expected); + + // Act + var actual = await _httpClient.SendAsync(_request); + + // Assert + Assert.Same(expected, actual); + _telemetryService.Verify(x => x.TrackDuration(It.IsAny(), It.IsAny>()), Times.Once); + + Assert.Equal(5, _properties.Count); + Assert.Equal(_request.Method.ToString(), _properties[TelemetryConstants.Method]); + Assert.Equal(_request.RequestUri.AbsoluteUri, _properties[TelemetryConstants.Uri]); + Assert.Equal(((int)expected.StatusCode).ToString(), _properties[TelemetryConstants.StatusCode]); + Assert.Equal("False", _properties[TelemetryConstants.Success]); + Assert.Equal(_properties[TelemetryConstants.ContentLength], expected.Content.Headers.ContentLength.Value.ToString()); + } + + [Fact] + public async Task EmitsTelemetryOnSuccessfulRequest() + { + // Arrange + var expected = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("Hello, world!"), + }; + _sendAsync = () => Task.FromResult(expected); + + // Act + var actual = await _httpClient.SendAsync(_request); + + // Assert + Assert.Same(expected, actual); + _telemetryService.Verify(x => x.TrackDuration(It.IsAny(), It.IsAny>()), Times.Once); + + Assert.Equal(5, _properties.Count); + Assert.Equal(_request.Method.ToString(), _properties[TelemetryConstants.Method]); + Assert.Equal(_request.RequestUri.AbsoluteUri, _properties[TelemetryConstants.Uri]); + Assert.Equal(((int)expected.StatusCode).ToString(), _properties[TelemetryConstants.StatusCode]); + Assert.Equal("True", _properties[TelemetryConstants.Success]); + Assert.Equal(_properties[TelemetryConstants.ContentLength], expected.Content.Headers.ContentLength.Value.ToString()); + } + + [Fact] + public async Task AllowsNoContentLength() + { + // Arrange + var expected = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StreamContent(new NoLengthStream()), + }; + _sendAsync = () => Task.FromResult(expected); + + // Act + var actual = await _httpClient.SendAsync(_request, HttpCompletionOption.ResponseHeadersRead); + + // Assert + Assert.Same(expected, actual); + _telemetryService.Verify(x => x.TrackDuration(It.IsAny(), It.IsAny>()), Times.Once); + + Assert.Equal(5, _properties.Count); + Assert.Equal(_request.Method.ToString(), _properties[TelemetryConstants.Method]); + Assert.Equal(_request.RequestUri.AbsoluteUri, _properties[TelemetryConstants.Uri]); + Assert.Equal(((int)expected.StatusCode).ToString(), _properties[TelemetryConstants.StatusCode]); + Assert.Equal("True", _properties[TelemetryConstants.Success]); + Assert.Equal("0", _properties[TelemetryConstants.ContentLength]); + } + + private class NoLengthStream : Stream + { + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() => throw new NotSupportedException(); + public override int Read(byte[] buffer, int offset, int count) => throw new NotImplementedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } + + private class FuncHttpMessageHandler : HttpMessageHandler + { + private readonly Func>> _getSendAsync; + + public FuncHttpMessageHandler(Func>> getSend) + { + _getSendAsync = getSend ?? throw new ArgumentNullException(nameof(getSend)); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + await Task.Yield(); + var sendAsync = _getSendAsync(); + return await sendAsync(); + } + } + } +} diff --git a/tests/CatalogTests/TestData/CatalogTestData.cs b/tests/CatalogTests/TestData/CatalogTestData.cs new file mode 100644 index 000000000..f2c4a1bb6 --- /dev/null +++ b/tests/CatalogTests/TestData/CatalogTestData.cs @@ -0,0 +1,824 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Newtonsoft.Json.Linq; +using NuGet.Services.Metadata.Catalog; + +namespace CatalogTests +{ + internal static class CatalogTestData + { + private const string _beforeIndex = @"{{ + ""@id"": ""{0}"", + ""@type"": [""CatalogRoot"", ""AppendOnlyCatalog"", ""Permalink""], + ""commitId"": ""fa2a4e80-aab1-434e-926a-6162704c34c8"", + ""commitTimeStamp"": ""2018-07-16T17:51:57.9718243Z"", + ""count"": 0, + ""nuget:lastCreated"": ""2018-07-16T17:23:04.453Z"", + ""nuget:lastDeleted"": ""2018-07-13T01:15:37Z"", + ""nuget:lastEdited"": ""2018-07-16T17:25:57.067Z"", + ""items"": [ + ], + ""@context"": {{ + ""@vocab"": ""http://schema.nuget.org/catalog#"", + ""nuget"": ""http://schema.nuget.org/schema#"", + ""items"": {{ + ""@id"": ""item"", + ""@container"": ""@set"" + }}, + ""parent"": {{ ""@type"": ""@id"" }}, + ""commitTimeStamp"": {{ ""@type"": ""http://www.w3.org/2001/XMLSchema#dateTime"" }}, + ""nuget:lastCreated"": {{ ""@type"": ""http://www.w3.org/2001/XMLSchema#dateTime"" }}, + ""nuget:lastEdited"": {{ ""@type"": ""http://www.w3.org/2001/XMLSchema#dateTime"" }}, + ""nuget:lastDeleted"": {{ ""@type"": ""http://www.w3.org/2001/XMLSchema#dateTime"" }} + }} +}}"; + + private const string _afterIndex = @"{{ + ""@id"": ""{0}"", + ""@type"": [ + ""CatalogRoot"", + ""AppendOnlyCatalog"", + ""Permalink"" + ], + ""commitId"": ""{1}"", + ""commitTimeStamp"": ""{2}"", + ""count"": 1, + ""nuget:lastCreated"": ""{3}"", + ""nuget:lastDeleted"": ""{4}"", + ""nuget:lastEdited"": ""{5}"", + ""items"": [ + {{ + ""@id"": ""{6}"", + ""@type"": ""CatalogPage"", + ""commitId"": ""{1}"", + ""commitTimeStamp"": ""{2}"", + ""count"": 1 + }} + ], + ""@context"": {{ + ""@vocab"": ""http://schema.nuget.org/catalog#"", + ""nuget"": ""http://schema.nuget.org/schema#"", + ""items"": {{ + ""@id"": ""item"", + ""@container"": ""@set"" + }}, + ""parent"": {{ + ""@type"": ""@id"" + }}, + ""commitTimeStamp"": {{ + ""@type"": ""http://www.w3.org/2001/XMLSchema#dateTime"" + }}, + ""nuget:lastCreated"": {{ + ""@type"": ""http://www.w3.org/2001/XMLSchema#dateTime"" + }}, + ""nuget:lastEdited"": {{ + ""@type"": ""http://www.w3.org/2001/XMLSchema#dateTime"" + }}, + ""nuget:lastDeleted"": {{ + ""@type"": ""http://www.w3.org/2001/XMLSchema#dateTime"" + }} + }} +}}"; + + private const string _page = @"{{ + ""@id"": ""{0}"", + ""@type"": ""CatalogPage"", + ""commitId"": ""{1}"", + ""commitTimeStamp"": ""{2}"", + ""count"": 1, + ""parent"": ""{3}"", + ""items"": [ + {{ + ""@id"": ""{4}"", + ""@type"": ""nuget:PackageDetails"", + ""commitId"": ""{1}"", + ""commitTimeStamp"": ""{2}"", + ""nuget:id"": ""Newtonsoft.Json"", + ""nuget:version"": ""9.0.2-beta1"" + }} + ], + ""@context"": {{ + ""@vocab"": ""http://schema.nuget.org/catalog#"", + ""nuget"": ""http://schema.nuget.org/schema#"", + ""items"": {{ + ""@id"": ""item"", + ""@container"": ""@set"" + }}, + ""parent"": {{ + ""@type"": ""@id"" + }}, + ""commitTimeStamp"": {{ + ""@type"": ""http://www.w3.org/2001/XMLSchema#dateTime"" + }}, + ""nuget:lastCreated"": {{ + ""@type"": ""http://www.w3.org/2001/XMLSchema#dateTime"" + }}, + ""nuget:lastEdited"": {{ + ""@type"": ""http://www.w3.org/2001/XMLSchema#dateTime"" + }}, + ""nuget:lastDeleted"": {{ + ""@type"": ""http://www.w3.org/2001/XMLSchema#dateTime"" + }} + }} +}}"; + + private const string _packageDetails = @"{{ + ""@id"": ""{0}"", + ""@type"": [ + ""PackageDetails"", + ""catalog:Permalink"" + ], + ""authors"": ""James Newton-King"", + ""catalog:commitId"": ""{1}"", + ""catalog:commitTimeStamp"": ""{2}"", + ""created"": ""{3}"",{6} + ""description"": ""Json.NET is a popular high-performance JSON framework for .NET"", + ""iconUrl"": ""http://www.newtonsoft.com/content/images/nugeticon.png"", + ""id"": ""Newtonsoft.Json"", + ""isPrerelease"": true, + ""language"": ""en-US"", + ""lastEdited"": ""{4}"", + ""licenseUrl"": ""https://raw.github.com/JamesNK/Newtonsoft.Json/master/LICENSE.md"", + ""listed"": true, + ""packageHash"": ""bq5DjCtCJpy9R5rsEeQlKz8qGF1Bh3wGaJKMlRwmCoKZ8WUCIFtU3JlyMOdAkSn66KCehCCAxMZFOQD4nNnH/w=="", + ""packageHashAlgorithm"": ""SHA512"", + ""packageSize"": 1871318, + ""projectUrl"": ""http://www.newtonsoft.com/json"", + ""published"": ""{5}"", + ""requireLicenseAcceptance"": false, + ""title"": ""Json.NET"", + ""verbatimVersion"": ""9.0.2-beta1"", + ""version"": ""9.0.2-beta1"", + ""dependencyGroups"": [ + {{ + ""@id"": ""{0}#dependencygroup/.netframework4.5"", + ""@type"": ""PackageDependencyGroup"", + ""targetFramework"": "".NETFramework4.5"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netframework4.0"", + ""@type"": ""PackageDependencyGroup"", + ""targetFramework"": "".NETFramework4.0"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netframework3.5"", + ""@type"": ""PackageDependencyGroup"", + ""targetFramework"": "".NETFramework3.5"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netframework2.0"", + ""@type"": ""PackageDependencyGroup"", + ""targetFramework"": "".NETFramework2.0"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netportable4.5-profile259"", + ""@type"": ""PackageDependencyGroup"", + ""targetFramework"": "".NETPortable4.5-Profile259"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netportable4.0-profile328"", + ""@type"": ""PackageDependencyGroup"", + ""targetFramework"": "".NETPortable4.0-Profile328"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1"", + ""@type"": ""PackageDependencyGroup"", + ""dependencies"": [ + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/microsoft.csharp"", + ""@type"": ""PackageDependency"", + ""id"": ""Microsoft.CSharp"", + ""range"": ""[4.0.1, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.collections"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Collections"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.diagnostics.debug"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Diagnostics.Debug"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.dynamic.runtime"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Dynamic.Runtime"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.globalization"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Globalization"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.io"", + ""@type"": ""PackageDependency"", + ""id"": ""System.IO"", + ""range"": ""[4.1.0, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.linq"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Linq"", + ""range"": ""[4.1.0, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.linq.expressions"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Linq.Expressions"", + ""range"": ""[4.1.0, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.objectmodel"", + ""@type"": ""PackageDependency"", + ""id"": ""System.ObjectModel"", + ""range"": ""[4.0.12, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.reflection"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Reflection"", + ""range"": ""[4.1.0, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.reflection.extensions"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Reflection.Extensions"", + ""range"": ""[4.0.1, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.resources.resourcemanager"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Resources.ResourceManager"", + ""range"": ""[4.0.1, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.runtime"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Runtime"", + ""range"": ""[4.1.0, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.runtime.extensions"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Runtime.Extensions"", + ""range"": ""[4.1.0, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.runtime.numerics"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Runtime.Numerics"", + ""range"": ""[4.0.1, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.runtime.serialization.primitives"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Runtime.Serialization.Primitives"", + ""range"": ""[4.1.1, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.text.encoding"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Text.Encoding"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.text.encoding.extensions"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Text.Encoding.Extensions"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.text.regularexpressions"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Text.RegularExpressions"", + ""range"": ""[4.1.0, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.threading"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Threading"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.threading.tasks"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Threading.Tasks"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.xml.readerwriter"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Xml.ReaderWriter"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.1/system.xml.xdocument"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Xml.XDocument"", + ""range"": ""[4.0.11, )"" + }} + ], + ""targetFramework"": "".NETStandard1.1"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0"", + ""@type"": ""PackageDependencyGroup"", + ""dependencies"": [ + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/microsoft.csharp"", + ""@type"": ""PackageDependency"", + ""id"": ""Microsoft.CSharp"", + ""range"": ""[4.0.1, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.collections"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Collections"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.diagnostics.debug"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Diagnostics.Debug"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.dynamic.runtime"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Dynamic.Runtime"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.globalization"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Globalization"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.io"", + ""@type"": ""PackageDependency"", + ""id"": ""System.IO"", + ""range"": ""[4.1.0, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.linq"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Linq"", + ""range"": ""[4.1.0, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.linq.expressions"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Linq.Expressions"", + ""range"": ""[4.1.0, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.objectmodel"", + ""@type"": ""PackageDependency"", + ""id"": ""System.ObjectModel"", + ""range"": ""[4.0.12, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.reflection"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Reflection"", + ""range"": ""[4.1.0, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.reflection.extensions"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Reflection.Extensions"", + ""range"": ""[4.0.1, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.resources.resourcemanager"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Resources.ResourceManager"", + ""range"": ""[4.0.1, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.runtime"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Runtime"", + ""range"": ""[4.1.0, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.runtime.extensions"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Runtime.Extensions"", + ""range"": ""[4.1.0, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.runtime.serialization.primitives"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Runtime.Serialization.Primitives"", + ""range"": ""[4.1.1, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.text.encoding"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Text.Encoding"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.text.encoding.extensions"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Text.Encoding.Extensions"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.text.regularexpressions"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Text.RegularExpressions"", + ""range"": ""[4.1.0, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.threading"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Threading"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.threading.tasks"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Threading.Tasks"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.xml.readerwriter"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Xml.ReaderWriter"", + ""range"": ""[4.0.11, )"" + }}, + {{ + ""@id"": ""{0}#dependencygroup/.netstandard1.0/system.xml.xdocument"", + ""@type"": ""PackageDependency"", + ""id"": ""System.Xml.XDocument"", + ""range"": ""[4.0.11, )"" + }} + ], + ""targetFramework"": "".NETStandard1.0"" + }} + ], + ""packageEntries"": [ + {{ + ""@id"": ""{0}#Newtonsoft.Json.nuspec"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 793, + ""fullName"": ""Newtonsoft.Json.nuspec"", + ""length"": 4359, + ""name"": ""Newtonsoft.Json.nuspec"" + }}, + {{ + ""@id"": ""{0}#lib/net20/Newtonsoft.Json.dll"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 195596, + ""fullName"": ""lib/net20/Newtonsoft.Json.dll"", + ""length"": 489984, + ""name"": ""Newtonsoft.Json.dll"" + }}, + {{ + ""@id"": ""{0}#lib/net20/Newtonsoft.Json.xml"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 47212, + ""fullName"": ""lib/net20/Newtonsoft.Json.xml"", + ""length"": 569142, + ""name"": ""Newtonsoft.Json.xml"" + }}, + {{ + ""@id"": ""{0}#lib/net35/Newtonsoft.Json.dll"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 183281, + ""fullName"": ""lib/net35/Newtonsoft.Json.dll"", + ""length"": 454144, + ""name"": ""Newtonsoft.Json.dll"" + }}, + {{ + ""@id"": ""{0}#lib/net35/Newtonsoft.Json.xml"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 42857, + ""fullName"": ""lib/net35/Newtonsoft.Json.xml"", + ""length"": 512044, + ""name"": ""Newtonsoft.Json.xml"" + }}, + {{ + ""@id"": ""{0}#lib/net40/Newtonsoft.Json.dll"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 200815, + ""fullName"": ""lib/net40/Newtonsoft.Json.dll"", + ""length"": 521728, + ""name"": ""Newtonsoft.Json.dll"" + }}, + {{ + ""@id"": ""{0}#lib/net40/Newtonsoft.Json.xml"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 44081, + ""fullName"": ""lib/net40/Newtonsoft.Json.xml"", + ""length"": 530291, + ""name"": ""Newtonsoft.Json.xml"" + }}, + {{ + ""@id"": ""{0}#lib/net45/Newtonsoft.Json.dll"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 203659, + ""fullName"": ""lib/net45/Newtonsoft.Json.dll"", + ""length"": 532992, + ""name"": ""Newtonsoft.Json.dll"" + }}, + {{ + ""@id"": ""{0}#lib/net45/Newtonsoft.Json.xml"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 42490, + ""fullName"": ""lib/net45/Newtonsoft.Json.xml"", + ""length"": 530291, + ""name"": ""Newtonsoft.Json.xml"" + }}, + {{ + ""@id"": ""{0}#lib/netstandard1.0/Newtonsoft.Json.dll"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 190807, + ""fullName"": ""lib/netstandard1.0/Newtonsoft.Json.dll"", + ""length"": 477184, + ""name"": ""Newtonsoft.Json.dll"" + }}, + {{ + ""@id"": ""{0}#lib/netstandard1.0/Newtonsoft.Json.xml"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 41114, + ""fullName"": ""lib/netstandard1.0/Newtonsoft.Json.xml"", + ""length"": 502998, + ""name"": ""Newtonsoft.Json.xml"" + }}, + {{ + ""@id"": ""{0}#lib/netstandard1.1/Newtonsoft.Json.dll"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 192147, + ""fullName"": ""lib/netstandard1.1/Newtonsoft.Json.dll"", + ""length"": 480768, + ""name"": ""Newtonsoft.Json.dll"" + }}, + {{ + ""@id"": ""{0}#lib/netstandard1.1/Newtonsoft.Json.xml"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 41114, + ""fullName"": ""lib/netstandard1.1/Newtonsoft.Json.xml"", + ""length"": 502998, + ""name"": ""Newtonsoft.Json.xml"" + }}, + {{ + ""@id"": ""{0}#lib/portable-net40%2Bsl5%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.dll"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 169713, + ""fullName"": ""lib/portable-net40%2Bsl5%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.dll"", + ""length"": 425984, + ""name"": ""Newtonsoft.Json.dll"" + }}, + {{ + ""@id"": ""{0}#lib/portable-net40%2Bsl5%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.xml"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 38591, + ""fullName"": ""lib/portable-net40%2Bsl5%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.xml"", + ""length"": 478486, + ""name"": ""Newtonsoft.Json.xml"" + }}, + {{ + ""@id"": ""{0}#lib/portable-net45%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.dll"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 190828, + ""fullName"": ""lib/portable-net45%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.dll"", + ""length"": 476672, + ""name"": ""Newtonsoft.Json.dll"" + }}, + {{ + ""@id"": ""{0}#lib/portable-net45%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.xml"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 40872, + ""fullName"": ""lib/portable-net45%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.xml"", + ""length"": 502998, + ""name"": ""Newtonsoft.Json.xml"" + }}, + {{ + ""@id"": ""{0}#tools/install.ps1"", + ""@type"": ""PackageEntry"", + ""compressedLength"": 1244, + ""fullName"": ""tools/install.ps1"", + ""length"": 3852, + ""name"": ""install.ps1"" + }} + ], + ""tags"": [ + ""json"" + ],{7} + ""@context"": {{ + ""@vocab"": ""http://schema.nuget.org/schema#"", + ""catalog"": ""http://schema.nuget.org/catalog#"", + ""xsd"": ""http://www.w3.org/2001/XMLSchema#"", + ""dependencies"": {{ + ""@id"": ""dependency"", + ""@container"": ""@set"" + }}, + ""dependencyGroups"": {{ + ""@id"": ""dependencyGroup"", + ""@container"": ""@set"" + }}, + ""packageEntries"": {{ + ""@id"": ""packageEntry"", + ""@container"": ""@set"" + }}, + ""packageTypes"": {{ + ""@id"": ""packageType"", + ""@container"": ""@set"" + }}, + ""supportedFrameworks"": {{ + ""@id"": ""supportedFramework"", + ""@container"": ""@set"" + }}, + ""tags"": {{ + ""@id"": ""tag"", + ""@container"": ""@set"" + }}, + ""vulnerabilities"": {{ + ""@id"": ""vulnerability"", + ""@container"": ""@set"" + }}, + ""published"": {{ + ""@type"": ""xsd:dateTime"" + }}, + ""created"": {{ + ""@type"": ""xsd:dateTime"" + }}, + ""lastEdited"": {{ + ""@type"": ""xsd:dateTime"" + }}, + ""catalog:commitTimeStamp"": {{ + ""@type"": ""xsd:dateTime"" + }}, + ""reasons"": {{ + ""@container"": ""@set"" + }} + }} +}}"; + + private const string _packageDeprecationDetails = @" + ""deprecation"": {{ + ""@id"": ""{0}#deprecation"",{1}{2} + ""reasons"": [{3}] + }},"; + + private const string _packageDeprecationAlternatePackageDetails = @" + ""alternatePackage"": {{ + ""@id"": ""{0}#deprecation/alternatePackage"", + ""id"": ""theId"", + ""range"": ""{1}"" + }},"; + + private const string _packageDeprecationMessageDetails = @" + ""message"": ""this is the message"","; + + private const string _packageVulnerabilitiesDetails = @" + ""vulnerabilities"": [{0}],"; + + private const string _packageVulnerabilityDetails = @" + {{ + ""@id"": ""{0}#vulnerability/GitHub/{1}"", + ""@type"": ""Vulnerability"", + ""advisoryUrl"": ""{2}"", + ""severity"": ""{3}"" + }}"; + + internal static JObject GetBeforeIndex(Uri indexUri) + { + return JObject.Parse(string.Format(_beforeIndex, indexUri)); + } + + internal static JObject GetAfterIndex( + Uri indexUri, + Guid commitId, + DateTime commitTimestamp, + DateTime lastCreated, + DateTime lastDeleted, + DateTime lastEdited, + Uri pageUri) + { + return JObject.Parse( + string.Format( + _afterIndex, + indexUri, + commitId.ToString(), + commitTimestamp.ToString("O"), + lastCreated.ToString("O"), + lastDeleted.ToString("O"), + lastEdited.ToString("O"), + pageUri)); + } + + internal static JObject GetPage( + Uri pageUri, + Guid commitId, + DateTime commitTimestamp, + Uri indexUri, + Uri packageDetailsUri) + { + return JObject.Parse( + string.Format( + _page, + pageUri, + commitId.ToString(), + commitTimestamp.ToString("O"), + indexUri, + packageDetailsUri)); + } + + internal static JObject GetPackageDetails( + Uri packageDetailsUri, + Guid commitId, + DateTime commitTimestamp, + DateTime created, + DateTime lastEdited, + DateTime published, + PackageDeprecationItem deprecation, + IList vulnerabilities) + { + return JObject.Parse( + string.Format( + _packageDetails, + packageDetailsUri, + commitId.ToString(), + commitTimestamp.ToString("O"), + created.ToString("O"), + lastEdited.ToString("O"), + published.ToString("O"), + GetPackageDeprecationDetails(packageDetailsUri, deprecation), + GetPackageVulnerabilityDetails(packageDetailsUri, vulnerabilities))); + } + + private static string GetPackageDeprecationDetails( + Uri packageDetailsUri, + PackageDeprecationItem deprecation) + { + if (deprecation == null) + { + return string.Empty; + } + + return string.Format( + _packageDeprecationDetails, + packageDetailsUri, + GetPackageDeprecationAlternatePackageDetails(packageDetailsUri, deprecation), + deprecation.Message == null ? string.Empty : _packageDeprecationMessageDetails, + string.Join(",", deprecation.Reasons.Select(r => $"\r\n \"{r}\"")) + "\r\n "); + } + + private static string GetPackageVulnerabilityDetails( + Uri packageDetailsUri, + IList vulnerabilities) + { + if (vulnerabilities == null) + { + return string.Empty; + } + + var vulnerabilityJson = string.Empty; + foreach (var vulnerability in vulnerabilities) + { + if (!string.IsNullOrEmpty(vulnerabilityJson)) + { + vulnerabilityJson += ","; + } + + vulnerabilityJson += string.Format( + _packageVulnerabilityDetails, + packageDetailsUri, + vulnerability.GitHubDatabaseKey, + vulnerability.AdvisoryUrl, + vulnerability.Severity); + } + + return string.Format(_packageVulnerabilitiesDetails, vulnerabilityJson); + } + + private static string GetPackageDeprecationAlternatePackageDetails( + Uri packageDetailsUri, + PackageDeprecationItem deprecation) + { + if (deprecation.AlternatePackageId == null) + { + return string.Empty; + } + + return string.Format( + _packageDeprecationAlternatePackageDetails, + packageDetailsUri, + deprecation.AlternatePackageRange); + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/DependencyMissingId.0.1.0.json b/tests/CatalogTests/TestData/DependencyMissingId.0.1.0.json new file mode 100644 index 000000000..1ab0d5f17 --- /dev/null +++ b/tests/CatalogTests/TestData/DependencyMissingId.0.1.0.json @@ -0,0 +1,87 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/dependencymissingid.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "DependencyMissingId", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "DependencyMissingId", + "id": "DependencyMissingId", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "0/GF3zhI84g+dL42+M6wFWPM/GEPuTMycMJcZL0CuekHaIbCJQvyogt5AXqcn3Dao+bu7mIFKEeEo243+rBXBw==", + "packageHashAlgorithm": "SHA512", + "packageSize": 416, + "published": "2017-01-03T08:15:00Z", + "title": "DependencyMissingId", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "dependencyGroups": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/dependencymissingid.0.1.0.json#dependencygroup", + "@type": "PackageDependencyGroup" + } + ], + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/dependencymissingid.0.1.0.json#DependencyMissingId.nuspec", + "@type": "PackageEntry", + "compressedLength": 230, + "fullName": "DependencyMissingId.nuspec", + "length": 450, + "name": "DependencyMissingId.nuspec" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/DependencyMissingId.0.1.0.nupkg b/tests/CatalogTests/TestData/DependencyMissingId.0.1.0.nupkg new file mode 100644 index 000000000..39d85f9ba Binary files /dev/null and b/tests/CatalogTests/TestData/DependencyMissingId.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestData/EmptyDependenciesElement.0.1.0.json b/tests/CatalogTests/TestData/EmptyDependenciesElement.0.1.0.json new file mode 100644 index 000000000..0c3224119 --- /dev/null +++ b/tests/CatalogTests/TestData/EmptyDependenciesElement.0.1.0.json @@ -0,0 +1,87 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencieselement.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "EmptyDependenciesElement", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "EmptyDependenciesElement", + "id": "EmptyDependenciesElement", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "DHe9khQhfffpCKdhACITsx3+gDDozHOob0payt61bDipMFWJmmWmtlI7mqwL/rEt76d+ghFpq/AiGLb0OSVraQ==", + "packageHashAlgorithm": "SHA512", + "packageSize": 417, + "published": "2017-01-03T08:15:00Z", + "title": "EmptyDependenciesElement", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "dependencyGroups": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencieselement.0.1.0.json#dependencygroup", + "@type": "PackageDependencyGroup" + } + ], + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencieselement.0.1.0.json#EmptyDependenciesElement.nuspec", + "@type": "PackageEntry", + "compressedLength": 221, + "fullName": "EmptyDependenciesElement.nuspec", + "length": 437, + "name": "EmptyDependenciesElement.nuspec" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/EmptyDependenciesElement.0.1.0.nupkg b/tests/CatalogTests/TestData/EmptyDependenciesElement.0.1.0.nupkg new file mode 100644 index 000000000..fd780c689 Binary files /dev/null and b/tests/CatalogTests/TestData/EmptyDependenciesElement.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestData/EmptyDependencyId.0.1.0.json b/tests/CatalogTests/TestData/EmptyDependencyId.0.1.0.json new file mode 100644 index 000000000..d2e47fe75 --- /dev/null +++ b/tests/CatalogTests/TestData/EmptyDependencyId.0.1.0.json @@ -0,0 +1,87 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencyid.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "EmptyDependencyId", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "EmptyDependencyId", + "id": "EmptyDependencyId", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "Q+pcY2TXQdAedDz4BDo4KXLjXYtT8KDKiGG+aXUqrhqH79NRYkmXBcKPxV7Uj2/ArcExf1CTUFU29Nm2OzCx6w==", + "packageHashAlgorithm": "SHA512", + "packageSize": 412, + "published": "2017-01-03T08:15:00Z", + "title": "EmptyDependencyId", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "dependencyGroups": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencyid.0.1.0.json#dependencygroup", + "@type": "PackageDependencyGroup" + } + ], + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencyid.0.1.0.json#EmptyDependencyId.nuspec", + "@type": "PackageEntry", + "compressedLength": 230, + "fullName": "EmptyDependencyId.nuspec", + "length": 448, + "name": "EmptyDependencyId.nuspec" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/EmptyDependencyId.0.1.0.nupkg b/tests/CatalogTests/TestData/EmptyDependencyId.0.1.0.nupkg new file mode 100644 index 000000000..c15543bbd Binary files /dev/null and b/tests/CatalogTests/TestData/EmptyDependencyId.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestData/EmptyDependencyIdWithGroups.0.1.0.json b/tests/CatalogTests/TestData/EmptyDependencyIdWithGroups.0.1.0.json new file mode 100644 index 000000000..c1b08a2cf --- /dev/null +++ b/tests/CatalogTests/TestData/EmptyDependencyIdWithGroups.0.1.0.json @@ -0,0 +1,101 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencyidwithgroups.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "EmptyDependencyIdWithGroups", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "EmptyDependencyIdWithGroups", + "id": "EmptyDependencyIdWithGroups", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "Y0eMlYdTHNz0F+wh/K1Jcf3MaC0QlbKPhpwnevt6yX63Po+coa/tXTYadfrbtoHUSFNSLLEP+OTqylRun6uAKQ==", + "packageHashAlgorithm": "SHA512", + "packageSize": 502, + "published": "2017-01-03T08:15:00Z", + "title": "EmptyDependencyIdWithGroups", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "dependencyGroups": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencyidwithgroups.0.1.0.json#dependencygroup/.netframework4.5", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencyidwithgroups.0.1.0.json#dependencygroup/.netframework4.5/newtonsoft.json", + "@type": "PackageDependency", + "id": "Newtonsoft.Json", + "range": "[10.0.1, )" + } + ], + "targetFramework": ".NETFramework4.5" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencyidwithgroups.0.1.0.json#dependencygroup/.netframework4.6", + "@type": "PackageDependencyGroup", + "targetFramework": ".NETFramework4.6" + } + ], + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencyidwithgroups.0.1.0.json#EmptyDependencyIdWithGroups.nuspec", + "@type": "PackageEntry", + "compressedLength": 300, + "fullName": "EmptyDependencyIdWithGroups.nuspec", + "length": 684, + "name": "EmptyDependencyIdWithGroups.nuspec" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/EmptyDependencyIdWithGroups.0.1.0.nupkg b/tests/CatalogTests/TestData/EmptyDependencyIdWithGroups.0.1.0.nupkg new file mode 100644 index 000000000..10135fd9f Binary files /dev/null and b/tests/CatalogTests/TestData/EmptyDependencyIdWithGroups.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestData/EmptyDependencyVersionRange.0.1.0.json b/tests/CatalogTests/TestData/EmptyDependencyVersionRange.0.1.0.json new file mode 100644 index 000000000..666437937 --- /dev/null +++ b/tests/CatalogTests/TestData/EmptyDependencyVersionRange.0.1.0.json @@ -0,0 +1,95 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencyversionrange.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "EmptyDependencyVersionRange", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "EmptyDependencyVersionRange", + "id": "EmptyDependencyVersionRange", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "J1NrM0aeDXk3kdjY3Aby8duP+mAt1JN/srVW5AMDIFXJXhWjZgmwAnAKBBrO6VDWXg2hi1x+uaWLHI0dUXraUg==", + "packageHashAlgorithm": "SHA512", + "packageSize": 450, + "published": "2017-01-03T08:15:00Z", + "title": "EmptyDependencyVersionRange", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "dependencyGroups": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencyversionrange.0.1.0.json#dependencygroup", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencyversionrange.0.1.0.json#dependencygroup/nuget.versioning", + "@type": "PackageDependency", + "id": "NuGet.Versioning", + "range": "" + } + ] + } + ], + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/emptydependencyversionrange.0.1.0.json#EmptyDependencyVersionRange.nuspec", + "@type": "PackageEntry", + "compressedLength": 248, + "fullName": "EmptyDependencyVersionRange.nuspec", + "length": 504, + "name": "EmptyDependencyVersionRange.nuspec" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/EmptyDependencyVersionRange.0.1.0.nupkg b/tests/CatalogTests/TestData/EmptyDependencyVersionRange.0.1.0.nupkg new file mode 100644 index 000000000..72de5fefd Binary files /dev/null and b/tests/CatalogTests/TestData/EmptyDependencyVersionRange.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestData/InvalidDependencyVersionRange.0.1.0.json b/tests/CatalogTests/TestData/InvalidDependencyVersionRange.0.1.0.json new file mode 100644 index 000000000..bf978de98 --- /dev/null +++ b/tests/CatalogTests/TestData/InvalidDependencyVersionRange.0.1.0.json @@ -0,0 +1,95 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/invaliddependencyversionrange.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "InvalidDependencyVersionRange", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "InvalidDependencyVersionRange", + "id": "InvalidDependencyVersionRange", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "RFCB7aV+m/vj0CYW3SmnLUd8jMYYJpOw+fz7hDNERlwUjz2rU+6lCVA2bdIko3YYc68jJN3n07S2OgKrUDwPBg==", + "packageHashAlgorithm": "SHA512", + "packageSize": 473, + "published": "2017-01-03T08:15:00Z", + "title": "InvalidDependencyVersionRange", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "dependencyGroups": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/invaliddependencyversionrange.0.1.0.json#dependencygroup", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/invaliddependencyversionrange.0.1.0.json#dependencygroup/nuget.versioning", + "@type": "PackageDependency", + "id": "NuGet.Versioning", + "range": "" + } + ] + } + ], + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/invaliddependencyversionrange.0.1.0.json#InvalidDependencyVersionRange.nuspec", + "@type": "PackageEntry", + "compressedLength": 267, + "fullName": "InvalidDependencyVersionRange.nuspec", + "length": 530, + "name": "InvalidDependencyVersionRange.nuspec" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/InvalidDependencyVersionRange.0.1.0.nupkg b/tests/CatalogTests/TestData/InvalidDependencyVersionRange.0.1.0.nupkg new file mode 100644 index 000000000..abbbb218f Binary files /dev/null and b/tests/CatalogTests/TestData/InvalidDependencyVersionRange.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestData/MissingDependencyVersionRange.0.1.0.json b/tests/CatalogTests/TestData/MissingDependencyVersionRange.0.1.0.json new file mode 100644 index 000000000..e06c3405c --- /dev/null +++ b/tests/CatalogTests/TestData/MissingDependencyVersionRange.0.1.0.json @@ -0,0 +1,95 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/missingdependencyversionrange.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "MissingDependencyVersionRange", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "MissingDependencyVersionRange", + "id": "MissingDependencyVersionRange", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "7K0E8VKZX8of+E4LU2gpnQbjA74nubVDhVDPLOdiRuYyMecuOu3eZ33QmVJ+ZsqLpZB/jnVEUUmUvL7sA4Y+JA==", + "packageHashAlgorithm": "SHA512", + "packageSize": 450, + "published": "2017-01-03T08:15:00Z", + "title": "MissingDependencyVersionRange", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "dependencyGroups": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/missingdependencyversionrange.0.1.0.json#dependencygroup", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/missingdependencyversionrange.0.1.0.json#dependencygroup/nuget.versioning", + "@type": "PackageDependency", + "id": "NuGet.Versioning", + "range": "" + } + ] + } + ], + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/missingdependencyversionrange.0.1.0.json#MissingDependencyVersionRange.nuspec", + "@type": "PackageEntry", + "compressedLength": 244, + "fullName": "MissingDependencyVersionRange.nuspec", + "length": 501, + "name": "MissingDependencyVersionRange.nuspec" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/MissingDependencyVersionRange.0.1.0.nupkg b/tests/CatalogTests/TestData/MissingDependencyVersionRange.0.1.0.nupkg new file mode 100644 index 000000000..b86910a32 Binary files /dev/null and b/tests/CatalogTests/TestData/MissingDependencyVersionRange.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestData/Newtonsoft.Json.9.0.2-beta1.json b/tests/CatalogTests/TestData/Newtonsoft.Json.9.0.2-beta1.json new file mode 100644 index 000000000..d4456ad60 --- /dev/null +++ b/tests/CatalogTests/TestData/Newtonsoft.Json.9.0.2-beta1.json @@ -0,0 +1,541 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "James Newton-King", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "Json.NET is a popular high-performance JSON framework for .NET", + "iconUrl": "http://www.newtonsoft.com/content/images/nugeticon.png", + "id": "Newtonsoft.Json", + "isPrerelease": true, + "language": "en-US", + "lastEdited": "2017-01-02T08:15:00Z", + "licenseUrl": "https://raw.github.com/JamesNK/Newtonsoft.Json/master/LICENSE.md", + "listed": true, + "packageHash": "bq5DjCtCJpy9R5rsEeQlKz8qGF1Bh3wGaJKMlRwmCoKZ8WUCIFtU3JlyMOdAkSn66KCehCCAxMZFOQD4nNnH/w==", + "packageHashAlgorithm": "SHA512", + "packageSize": 1871318, + "projectUrl": "http://www.newtonsoft.com/json", + "published": "2017-01-03T08:15:00Z", + "requireLicenseAcceptance": false, + "title": "Json.NET", + "verbatimVersion": "9.0.2-beta1", + "version": "9.0.2-beta1", + "dependencyGroups": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netframework4.5", + "@type": "PackageDependencyGroup", + "targetFramework": ".NETFramework4.5" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netframework4.0", + "@type": "PackageDependencyGroup", + "targetFramework": ".NETFramework4.0" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netframework3.5", + "@type": "PackageDependencyGroup", + "targetFramework": ".NETFramework3.5" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netframework2.0", + "@type": "PackageDependencyGroup", + "targetFramework": ".NETFramework2.0" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netportable4.5-profile259", + "@type": "PackageDependencyGroup", + "targetFramework": ".NETPortable4.5-Profile259" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netportable4.0-profile328", + "@type": "PackageDependencyGroup", + "targetFramework": ".NETPortable4.0-Profile328" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/microsoft.csharp", + "@type": "PackageDependency", + "id": "Microsoft.CSharp", + "range": "[4.0.1, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.collections", + "@type": "PackageDependency", + "id": "System.Collections", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.diagnostics.debug", + "@type": "PackageDependency", + "id": "System.Diagnostics.Debug", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.dynamic.runtime", + "@type": "PackageDependency", + "id": "System.Dynamic.Runtime", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.globalization", + "@type": "PackageDependency", + "id": "System.Globalization", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.io", + "@type": "PackageDependency", + "id": "System.IO", + "range": "[4.1.0, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.linq", + "@type": "PackageDependency", + "id": "System.Linq", + "range": "[4.1.0, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.linq.expressions", + "@type": "PackageDependency", + "id": "System.Linq.Expressions", + "range": "[4.1.0, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.objectmodel", + "@type": "PackageDependency", + "id": "System.ObjectModel", + "range": "[4.0.12, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.reflection", + "@type": "PackageDependency", + "id": "System.Reflection", + "range": "[4.1.0, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.reflection.extensions", + "@type": "PackageDependency", + "id": "System.Reflection.Extensions", + "range": "[4.0.1, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.resources.resourcemanager", + "@type": "PackageDependency", + "id": "System.Resources.ResourceManager", + "range": "[4.0.1, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.runtime", + "@type": "PackageDependency", + "id": "System.Runtime", + "range": "[4.1.0, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.runtime.extensions", + "@type": "PackageDependency", + "id": "System.Runtime.Extensions", + "range": "[4.1.0, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.runtime.numerics", + "@type": "PackageDependency", + "id": "System.Runtime.Numerics", + "range": "[4.0.1, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.runtime.serialization.primitives", + "@type": "PackageDependency", + "id": "System.Runtime.Serialization.Primitives", + "range": "[4.1.1, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.text.encoding", + "@type": "PackageDependency", + "id": "System.Text.Encoding", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.text.encoding.extensions", + "@type": "PackageDependency", + "id": "System.Text.Encoding.Extensions", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.text.regularexpressions", + "@type": "PackageDependency", + "id": "System.Text.RegularExpressions", + "range": "[4.1.0, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.threading", + "@type": "PackageDependency", + "id": "System.Threading", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.threading.tasks", + "@type": "PackageDependency", + "id": "System.Threading.Tasks", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.xml.readerwriter", + "@type": "PackageDependency", + "id": "System.Xml.ReaderWriter", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.1/system.xml.xdocument", + "@type": "PackageDependency", + "id": "System.Xml.XDocument", + "range": "[4.0.11, )" + } + ], + "targetFramework": ".NETStandard1.1" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/microsoft.csharp", + "@type": "PackageDependency", + "id": "Microsoft.CSharp", + "range": "[4.0.1, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.collections", + "@type": "PackageDependency", + "id": "System.Collections", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.diagnostics.debug", + "@type": "PackageDependency", + "id": "System.Diagnostics.Debug", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.dynamic.runtime", + "@type": "PackageDependency", + "id": "System.Dynamic.Runtime", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.globalization", + "@type": "PackageDependency", + "id": "System.Globalization", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.io", + "@type": "PackageDependency", + "id": "System.IO", + "range": "[4.1.0, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.linq", + "@type": "PackageDependency", + "id": "System.Linq", + "range": "[4.1.0, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.linq.expressions", + "@type": "PackageDependency", + "id": "System.Linq.Expressions", + "range": "[4.1.0, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.objectmodel", + "@type": "PackageDependency", + "id": "System.ObjectModel", + "range": "[4.0.12, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.reflection", + "@type": "PackageDependency", + "id": "System.Reflection", + "range": "[4.1.0, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.reflection.extensions", + "@type": "PackageDependency", + "id": "System.Reflection.Extensions", + "range": "[4.0.1, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.resources.resourcemanager", + "@type": "PackageDependency", + "id": "System.Resources.ResourceManager", + "range": "[4.0.1, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.runtime", + "@type": "PackageDependency", + "id": "System.Runtime", + "range": "[4.1.0, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.runtime.extensions", + "@type": "PackageDependency", + "id": "System.Runtime.Extensions", + "range": "[4.1.0, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.runtime.serialization.primitives", + "@type": "PackageDependency", + "id": "System.Runtime.Serialization.Primitives", + "range": "[4.1.1, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.text.encoding", + "@type": "PackageDependency", + "id": "System.Text.Encoding", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.text.encoding.extensions", + "@type": "PackageDependency", + "id": "System.Text.Encoding.Extensions", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.text.regularexpressions", + "@type": "PackageDependency", + "id": "System.Text.RegularExpressions", + "range": "[4.1.0, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.threading", + "@type": "PackageDependency", + "id": "System.Threading", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.threading.tasks", + "@type": "PackageDependency", + "id": "System.Threading.Tasks", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.xml.readerwriter", + "@type": "PackageDependency", + "id": "System.Xml.ReaderWriter", + "range": "[4.0.11, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#dependencygroup/.netstandard1.0/system.xml.xdocument", + "@type": "PackageDependency", + "id": "System.Xml.XDocument", + "range": "[4.0.11, )" + } + ], + "targetFramework": ".NETStandard1.0" + } + ], + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#Newtonsoft.Json.nuspec", + "@type": "PackageEntry", + "compressedLength": 793, + "fullName": "Newtonsoft.Json.nuspec", + "length": 4359, + "name": "Newtonsoft.Json.nuspec" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/net20/Newtonsoft.Json.dll", + "@type": "PackageEntry", + "compressedLength": 195596, + "fullName": "lib/net20/Newtonsoft.Json.dll", + "length": 489984, + "name": "Newtonsoft.Json.dll" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/net20/Newtonsoft.Json.xml", + "@type": "PackageEntry", + "compressedLength": 47212, + "fullName": "lib/net20/Newtonsoft.Json.xml", + "length": 569142, + "name": "Newtonsoft.Json.xml" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/net35/Newtonsoft.Json.dll", + "@type": "PackageEntry", + "compressedLength": 183281, + "fullName": "lib/net35/Newtonsoft.Json.dll", + "length": 454144, + "name": "Newtonsoft.Json.dll" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/net35/Newtonsoft.Json.xml", + "@type": "PackageEntry", + "compressedLength": 42857, + "fullName": "lib/net35/Newtonsoft.Json.xml", + "length": 512044, + "name": "Newtonsoft.Json.xml" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/net40/Newtonsoft.Json.dll", + "@type": "PackageEntry", + "compressedLength": 200815, + "fullName": "lib/net40/Newtonsoft.Json.dll", + "length": 521728, + "name": "Newtonsoft.Json.dll" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/net40/Newtonsoft.Json.xml", + "@type": "PackageEntry", + "compressedLength": 44081, + "fullName": "lib/net40/Newtonsoft.Json.xml", + "length": 530291, + "name": "Newtonsoft.Json.xml" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/net45/Newtonsoft.Json.dll", + "@type": "PackageEntry", + "compressedLength": 203659, + "fullName": "lib/net45/Newtonsoft.Json.dll", + "length": 532992, + "name": "Newtonsoft.Json.dll" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/net45/Newtonsoft.Json.xml", + "@type": "PackageEntry", + "compressedLength": 42490, + "fullName": "lib/net45/Newtonsoft.Json.xml", + "length": 530291, + "name": "Newtonsoft.Json.xml" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/netstandard1.0/Newtonsoft.Json.dll", + "@type": "PackageEntry", + "compressedLength": 190807, + "fullName": "lib/netstandard1.0/Newtonsoft.Json.dll", + "length": 477184, + "name": "Newtonsoft.Json.dll" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/netstandard1.0/Newtonsoft.Json.xml", + "@type": "PackageEntry", + "compressedLength": 41114, + "fullName": "lib/netstandard1.0/Newtonsoft.Json.xml", + "length": 502998, + "name": "Newtonsoft.Json.xml" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/netstandard1.1/Newtonsoft.Json.dll", + "@type": "PackageEntry", + "compressedLength": 192147, + "fullName": "lib/netstandard1.1/Newtonsoft.Json.dll", + "length": 480768, + "name": "Newtonsoft.Json.dll" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/netstandard1.1/Newtonsoft.Json.xml", + "@type": "PackageEntry", + "compressedLength": 41114, + "fullName": "lib/netstandard1.1/Newtonsoft.Json.xml", + "length": 502998, + "name": "Newtonsoft.Json.xml" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/portable-net40%2Bsl5%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.dll", + "@type": "PackageEntry", + "compressedLength": 169713, + "fullName": "lib/portable-net40%2Bsl5%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.dll", + "length": 425984, + "name": "Newtonsoft.Json.dll" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/portable-net40%2Bsl5%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.xml", + "@type": "PackageEntry", + "compressedLength": 38591, + "fullName": "lib/portable-net40%2Bsl5%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.xml", + "length": 478486, + "name": "Newtonsoft.Json.xml" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/portable-net45%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.dll", + "@type": "PackageEntry", + "compressedLength": 190828, + "fullName": "lib/portable-net45%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.dll", + "length": 476672, + "name": "Newtonsoft.Json.dll" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#lib/portable-net45%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.xml", + "@type": "PackageEntry", + "compressedLength": 40872, + "fullName": "lib/portable-net45%2Bwp80%2Bwin8%2Bwpa81/Newtonsoft.Json.xml", + "length": 502998, + "name": "Newtonsoft.Json.xml" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/newtonsoft.json.9.0.2-beta1.json#tools/install.ps1", + "@type": "PackageEntry", + "compressedLength": 1244, + "fullName": "tools/install.ps1", + "length": 3852, + "name": "install.ps1" + } + ], + "tags": [ + "json" + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/Newtonsoft.Json.9.0.2-beta1.nupkg b/tests/CatalogTests/TestData/Newtonsoft.Json.9.0.2-beta1.nupkg new file mode 100644 index 000000000..2a96df9f4 Binary files /dev/null and b/tests/CatalogTests/TestData/Newtonsoft.Json.9.0.2-beta1.nupkg differ diff --git a/tests/CatalogTests/TestData/OneValidDependencyOneEmptyId.0.1.0.json b/tests/CatalogTests/TestData/OneValidDependencyOneEmptyId.0.1.0.json new file mode 100644 index 000000000..6f6081ca2 --- /dev/null +++ b/tests/CatalogTests/TestData/OneValidDependencyOneEmptyId.0.1.0.json @@ -0,0 +1,95 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/onevaliddependencyoneemptyid.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "OneValidDependencyOneEmptyId", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "OneValidDependencyOneEmptyId", + "id": "OneValidDependencyOneEmptyId", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "0qvYQHM5EL9WuZMAiiHgj6FDAmcuaNAgbrlI29XothTBJf2ksXfQmSsRN30q9YnfEiL9LJl5uTJP9RZixybc6Q==", + "packageHashAlgorithm": "SHA512", + "packageSize": 465, + "published": "2017-01-03T08:15:00Z", + "title": "OneValidDependencyOneEmptyId", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "dependencyGroups": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/onevaliddependencyoneemptyid.0.1.0.json#dependencygroup", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/onevaliddependencyoneemptyid.0.1.0.json#dependencygroup/newtonsoft.json", + "@type": "PackageDependency", + "id": "Newtonsoft.Json", + "range": "[9.0.1, )" + } + ] + } + ], + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/onevaliddependencyoneemptyid.0.1.0.json#OneValidDependencyOneEmptyId.nuspec", + "@type": "PackageEntry", + "compressedLength": 261, + "fullName": "OneValidDependencyOneEmptyId.nuspec", + "length": 551, + "name": "OneValidDependencyOneEmptyId.nuspec" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/OneValidDependencyOneEmptyId.0.1.0.nupkg b/tests/CatalogTests/TestData/OneValidDependencyOneEmptyId.0.1.0.nupkg new file mode 100644 index 000000000..e54a31ea5 Binary files /dev/null and b/tests/CatalogTests/TestData/OneValidDependencyOneEmptyId.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestData/OneValidDependencyOneEmptyIdWithGroups.0.1.0.json b/tests/CatalogTests/TestData/OneValidDependencyOneEmptyIdWithGroups.0.1.0.json new file mode 100644 index 000000000..2c5426d40 --- /dev/null +++ b/tests/CatalogTests/TestData/OneValidDependencyOneEmptyIdWithGroups.0.1.0.json @@ -0,0 +1,109 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/onevaliddependencyoneemptyidwithgroups.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "OneValidDependencyOneEmptyIdWithGroups", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "OneValidDependencyOneEmptyIdWithGroups", + "id": "OneValidDependencyOneEmptyIdWithGroups", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "iPhzAJ5o9SvaQm51Aee8GfaWYVWHxtXfBttzX/QHV1SffEL6RyeC4DioYEsmlyw8gCAZ46n3ArVEaYJs3Wvusg==", + "packageHashAlgorithm": "SHA512", + "packageSize": 540, + "published": "2017-01-03T08:15:00Z", + "title": "OneValidDependencyOneEmptyIdWithGroups", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "dependencyGroups": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/onevaliddependencyoneemptyidwithgroups.0.1.0.json#dependencygroup/.netframework4.5", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/onevaliddependencyoneemptyidwithgroups.0.1.0.json#dependencygroup/.netframework4.5/newtonsoft.json", + "@type": "PackageDependency", + "id": "Newtonsoft.Json", + "range": "[10.0.1, )" + } + ], + "targetFramework": ".NETFramework4.5" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/onevaliddependencyoneemptyidwithgroups.0.1.0.json#dependencygroup/.netframework4.6", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/onevaliddependencyoneemptyidwithgroups.0.1.0.json#dependencygroup/.netframework4.6/newtonsoft.json", + "@type": "PackageDependency", + "id": "Newtonsoft.Json", + "range": "[10.0.2, )" + } + ], + "targetFramework": ".NETFramework4.6" + } + ], + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/onevaliddependencyoneemptyidwithgroups.0.1.0.json#OneValidDependencyOneEmptyIdWithGroups.nuspec", + "@type": "PackageEntry", + "compressedLength": 316, + "fullName": "OneValidDependencyOneEmptyIdWithGroups.nuspec", + "length": 790, + "name": "OneValidDependencyOneEmptyIdWithGroups.nuspec" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/OneValidDependencyOneEmptyIdWithGroups.0.1.0.nupkg b/tests/CatalogTests/TestData/OneValidDependencyOneEmptyIdWithGroups.0.1.0.nupkg new file mode 100644 index 000000000..4590b987a Binary files /dev/null and b/tests/CatalogTests/TestData/OneValidDependencyOneEmptyIdWithGroups.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestData/PackageTypeCollapseDuplicate.json b/tests/CatalogTests/TestData/PackageTypeCollapseDuplicate.json new file mode 100644 index 000000000..7a97dc312 --- /dev/null +++ b/tests/CatalogTests/TestData/PackageTypeCollapseDuplicate.json @@ -0,0 +1,88 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/packagetypecollapseduplicate.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "PackageTypeCollapseDuplicate", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "PackageTypeCollapseDuplicate", + "id": "PackageTypeCollapseDuplicate", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "ffDPvjkuh1a2aPINTsJALkUvIqYd3AcymgG432U9HWZDCgNuzUw9CswhhaGF9lJjvjb7BOhI24QA9IhYZ7qjHw==", + "packageHashAlgorithm": "SHA512", + "packageSize": 408, + "published": "2017-01-03T08:15:00Z", + "title": "PackageTypeCollapseDuplicate", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypecollapseduplicate.0.1.0.json#PackageTypeCollapseDuplicate.nuspec", + "@type": "PackageEntry", + "compressedLength": 240, + "fullName": "PackageTypeCollapseDuplicate.nuspec", + "length": 525, + "name": "PackageTypeCollapseDuplicate.nuspec" + } + ], + "packageTypes": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypecollapseduplicate.0.1.0.json#packagetypes/type1", + "@type": "PackageType", + "name": "type1" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/PackageTypeCollapseDuplicate.nupkg b/tests/CatalogTests/TestData/PackageTypeCollapseDuplicate.nupkg new file mode 100644 index 000000000..52599b5c6 Binary files /dev/null and b/tests/CatalogTests/TestData/PackageTypeCollapseDuplicate.nupkg differ diff --git a/tests/CatalogTests/TestData/PackageTypeMultiple.json b/tests/CatalogTests/TestData/PackageTypeMultiple.json new file mode 100644 index 000000000..7f06f8a98 --- /dev/null +++ b/tests/CatalogTests/TestData/PackageTypeMultiple.json @@ -0,0 +1,93 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/packagetypemultiple.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "PackageTypeMultiple", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "PackageTypeMultiple", + "id": "PackageTypeMultiple", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "0HkAz8TNHlerRU1aG/Vh9Urh7DEZ6NamqGKT+CGLY1ImM9q/0vFWP0Q+Kouvq3NcW0uaoVHtL5ewCaxTa20Gdw==", + "packageHashAlgorithm": "SHA512", + "packageSize": 385, + "published": "2017-01-03T08:15:00Z", + "title": "PackageTypeMultiple", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypemultiple.0.1.0.json#PackageTypeMultiple.nuspec", + "@type": "PackageEntry", + "compressedLength": 235, + "fullName": "PackageTypeMultiple.nuspec", + "length": 489, + "name": "PackageTypeMultiple.nuspec" + } + ], + "packageTypes": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypemultiple.0.1.0.json#packagetypes/type1", + "@type": "PackageType", + "name": "type1" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypemultiple.0.1.0.json#packagetypes/type2", + "@type": "PackageType", + "name": "type2" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/PackageTypeMultiple.nupkg b/tests/CatalogTests/TestData/PackageTypeMultiple.nupkg new file mode 100644 index 000000000..96065940a Binary files /dev/null and b/tests/CatalogTests/TestData/PackageTypeMultiple.nupkg differ diff --git a/tests/CatalogTests/TestData/PackageTypeMultipleTypesNodes.json b/tests/CatalogTests/TestData/PackageTypeMultipleTypesNodes.json new file mode 100644 index 000000000..5144adda8 --- /dev/null +++ b/tests/CatalogTests/TestData/PackageTypeMultipleTypesNodes.json @@ -0,0 +1,93 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/packagetypemultipletypesnodes.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "PackageTypeMultipleTypesNodes", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "PackageTypeMultipleTypesNodes", + "id": "PackageTypeMultipleTypesNodes", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "Ubc0q7yW2DQjfPiabhKZShOIBWWsRJr8yhCuDEe1OIqbaObOFnaM5uIkT+MdHYFL51X6IfJHnGNlpMp+lgDDnA==", + "packageHashAlgorithm": "SHA512", + "packageSize": 414, + "published": "2017-01-03T08:15:00Z", + "title": "PackageTypeMultipleTypesNodes", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypemultipletypesnodes.0.1.0.json#PackageTypeMultipleTypesNodes.nuspec", + "@type": "PackageEntry", + "compressedLength": 244, + "fullName": "PackageTypeMultipleTypesNodes.nuspec", + "length": 570, + "name": "PackageTypeMultipleTypesNodes.nuspec" + } + ], + "packageTypes": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypemultipletypesnodes.0.1.0.json#packagetypes/type1", + "@type": "PackageType", + "name": "type1" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypemultipletypesnodes.0.1.0.json#packagetypes/type2", + "@type": "PackageType", + "name": "type2" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/PackageTypeMultipleTypesNodes.nupkg b/tests/CatalogTests/TestData/PackageTypeMultipleTypesNodes.nupkg new file mode 100644 index 000000000..73de0dfb9 Binary files /dev/null and b/tests/CatalogTests/TestData/PackageTypeMultipleTypesNodes.nupkg differ diff --git a/tests/CatalogTests/TestData/PackageTypeSameTypeDifferentCase.json b/tests/CatalogTests/TestData/PackageTypeSameTypeDifferentCase.json new file mode 100644 index 000000000..c7176cfad --- /dev/null +++ b/tests/CatalogTests/TestData/PackageTypeSameTypeDifferentCase.json @@ -0,0 +1,93 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesametypedifferentcase.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "PackageTypeSameTypeDifferentCase", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "PackageTypeSameTypeDifferentCase", + "id": "PackageTypeSameTypeDifferentCase", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "jc4IbB2R8D7OU6OB0C18HnGCfJQQMJa+WoV/+hCpP2o4u3W7N34t0rD81Q8OQCGnSJFFszy2WgFEAQydRHtfyw==", + "packageHashAlgorithm": "SHA512", + "packageSize": 421, + "published": "2017-01-03T08:15:00Z", + "title": "PackageTypeSameTypeDifferentCase", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesametypedifferentcase.0.1.0.json#PackageTypeSameTypeDifferentCase.nuspec", + "@type": "PackageEntry", + "compressedLength": 245, + "fullName": "PackageTypeSameTypeDifferentCase.nuspec", + "length": 541, + "name": "PackageTypeSameTypeDifferentCase.nuspec" + } + ], + "packageTypes": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesametypedifferentcase.0.1.0.json#packagetypes/type1", + "@type": "PackageType", + "name": "type1" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesametypedifferentcase.0.1.0.json#packagetypes/Type1", + "@type": "PackageType", + "name": "Type1" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/PackageTypeSameTypeDifferentCase.nupkg b/tests/CatalogTests/TestData/PackageTypeSameTypeDifferentCase.nupkg new file mode 100644 index 000000000..14ce5d4fa Binary files /dev/null and b/tests/CatalogTests/TestData/PackageTypeSameTypeDifferentCase.nupkg differ diff --git a/tests/CatalogTests/TestData/PackageTypeSameTypeDifferentVersionType.json b/tests/CatalogTests/TestData/PackageTypeSameTypeDifferentVersionType.json new file mode 100644 index 000000000..5caf215f3 --- /dev/null +++ b/tests/CatalogTests/TestData/PackageTypeSameTypeDifferentVersionType.json @@ -0,0 +1,101 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesametypedifferentversiontype.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "PackageTypeSameTypeDifferentVersionType", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "PackageTypeSameTypeDifferentVersionType", + "id": "PackageTypeSameTypeDifferentVersionType", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "YFOfxt+HLsc5eURjois2N+FBxreFIAfFCx2uXeBcE0zZ+KEKedXfBhIhIKeEK0F+0uhgsPmI6XJNe4gLSEfwng==", + "packageHashAlgorithm": "SHA512", + "packageSize": 440, + "published": "2017-01-03T08:15:00Z", + "title": "PackageTypeSameTypeDifferentVersionType", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesametypedifferentversiontype.0.1.0.json#PackageTypeSameTypeDifferentVersionType.nuspec", + "@type": "PackageEntry", + "compressedLength": 250, + "fullName": "PackageTypeSameTypeDifferentVersionType.nuspec", + "length": 653, + "name": "PackageTypeSameTypeDifferentVersionType.nuspec" + } + ], + "packageTypes": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesametypedifferentversiontype.0.1.0.json#packagetypes/type1/1.0", + "@type": "PackageType", + "name": "type1", + "version": "1.0" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesametypedifferentversiontype.0.1.0.json#packagetypes/type1/1.0.0", + "@type": "PackageType", + "name": "type1", + "version": "1.0.0" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesametypedifferentversiontype.0.1.0.json#packagetypes/type1/1.0.0.0", + "@type": "PackageType", + "name": "type1", + "version": "1.0.0.0" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/PackageTypeSameTypeDifferentVersionType.nupkg b/tests/CatalogTests/TestData/PackageTypeSameTypeDifferentVersionType.nupkg new file mode 100644 index 000000000..28149ebb0 Binary files /dev/null and b/tests/CatalogTests/TestData/PackageTypeSameTypeDifferentVersionType.nupkg differ diff --git a/tests/CatalogTests/TestData/PackageTypeSameTypeTwoVersion.json b/tests/CatalogTests/TestData/PackageTypeSameTypeTwoVersion.json new file mode 100644 index 000000000..cb44eab45 --- /dev/null +++ b/tests/CatalogTests/TestData/PackageTypeSameTypeTwoVersion.json @@ -0,0 +1,95 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesametypetwoversion.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "PackageTypeSameTypeTwoVersion", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "PackageTypeSameTypeTwoVersion", + "id": "PackageTypeSameTypeTwoVersion", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "H2QAIBh5LPHHEjI/ihxo0tKgXW8qRBGwp9T4zIzIB4wU3eIbV+PsQAfws4eSiksipQOzN0mhCCT1Sei9lF4HEg==", + "packageHashAlgorithm": "SHA512", + "packageSize": 415, + "published": "2017-01-03T08:15:00Z", + "title": "PackageTypeSameTypeTwoVersion", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesametypetwoversion.0.1.0.json#PackageTypeSameTypeTwoVersion.nuspec", + "@type": "PackageEntry", + "compressedLength": 245, + "fullName": "PackageTypeSameTypeTwoVersion.nuspec", + "length": 561, + "name": "PackageTypeSameTypeTwoVersion.nuspec" + } + ], + "packageTypes": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesametypetwoversion.0.1.0.json#packagetypes/type1/1.0.0", + "@type": "PackageType", + "name": "type1", + "version": "1.0.0" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesametypetwoversion.0.1.0.json#packagetypes/type1/2.0.0", + "@type": "PackageType", + "name": "type1", + "version": "2.0.0" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/PackageTypeSameTypeTwoVersion.nupkg b/tests/CatalogTests/TestData/PackageTypeSameTypeTwoVersion.nupkg new file mode 100644 index 000000000..3dc6a6fbe Binary files /dev/null and b/tests/CatalogTests/TestData/PackageTypeSameTypeTwoVersion.nupkg differ diff --git a/tests/CatalogTests/TestData/PackageTypeSingle.json b/tests/CatalogTests/TestData/PackageTypeSingle.json new file mode 100644 index 000000000..3be015250 --- /dev/null +++ b/tests/CatalogTests/TestData/PackageTypeSingle.json @@ -0,0 +1,88 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesingle.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "PackageTypeSingle", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "PackageTypeSingle", + "id": "PackageTypeSingle", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "N4tTs/W/KSAeLOzgI9SLETafyYmJS4S4JM0TtMLJUHs6SBKmNd0jxOh+T+WBUNbsw/Pg3o+pScGuica1He8ykw==", + "packageHashAlgorithm": "SHA512", + "packageSize": 374, + "published": "2017-01-03T08:15:00Z", + "title": "PackageTypeSingle", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesingle.0.1.0.json#PackageTypeSingle.nuspec", + "@type": "PackageEntry", + "compressedLength": 228, + "fullName": "PackageTypeSingle.nuspec", + "length": 445, + "name": "PackageTypeSingle.nuspec" + } + ], + "packageTypes": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesingle.0.1.0.json#packagetypes/type1", + "@type": "PackageType", + "name": "type1" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/PackageTypeSingle.nupkg b/tests/CatalogTests/TestData/PackageTypeSingle.nupkg new file mode 100644 index 000000000..08e12c3af Binary files /dev/null and b/tests/CatalogTests/TestData/PackageTypeSingle.nupkg differ diff --git a/tests/CatalogTests/TestData/PackageTypeSingleWithVersion.json b/tests/CatalogTests/TestData/PackageTypeSingleWithVersion.json new file mode 100644 index 000000000..e1d535fbf --- /dev/null +++ b/tests/CatalogTests/TestData/PackageTypeSingleWithVersion.json @@ -0,0 +1,89 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesinglewithversion.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "PackageTypeSingleWithVersion", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "PackageTypeSingleWithVersion", + "id": "PackageTypeSingleWithVersion", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "N0vmLYi8LFJ3BcVSdoMfsABeMTF7va0Ig7KV/o6rtoFlaLncFC+bKn9wtuI0Ft8V69eUKcRxIyZPcxKiaC+P6w==", + "packageHashAlgorithm": "SHA512", + "packageSize": 409, + "published": "2017-01-03T08:15:00Z", + "title": "PackageTypeSingleWithVersion", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesinglewithversion.0.1.0.json#PackageTypeSingleWithVersion.nuspec", + "@type": "PackageEntry", + "compressedLength": 241, + "fullName": "PackageTypeSingleWithVersion.nuspec", + "length": 505, + "name": "PackageTypeSingleWithVersion.nuspec" + } + ], + "packageTypes": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypesinglewithversion.0.1.0.json#packagetypes/type1/1.0.0", + "@type": "PackageType", + "name": "type1", + "version": "1.0.0" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/PackageTypeSingleWithVersion.nupkg b/tests/CatalogTests/TestData/PackageTypeSingleWithVersion.nupkg new file mode 100644 index 000000000..43673fb1f Binary files /dev/null and b/tests/CatalogTests/TestData/PackageTypeSingleWithVersion.nupkg differ diff --git a/tests/CatalogTests/TestData/PackageTypeWhiteSpace.json b/tests/CatalogTests/TestData/PackageTypeWhiteSpace.json new file mode 100644 index 000000000..7799b08c1 --- /dev/null +++ b/tests/CatalogTests/TestData/PackageTypeWhiteSpace.json @@ -0,0 +1,88 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/packagetypewhitespace.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "PackageTypeWhiteSpace", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "PackageTypeWhiteSpace", + "id": "PackageTypeWhiteSpace", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "33xLZngDUJD4Di8s8gUc65RtbOrLR9xcUqJpo4Un/u5clYVhPSQL846VwLQEeDW3lXVgjjN3KQdxMyaSVzQ+DQ==", + "packageHashAlgorithm": "SHA512", + "packageSize": 386, + "published": "2017-01-03T08:15:00Z", + "title": "PackageTypeWhiteSpace", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypewhitespace.0.1.0.json#PackageTypeWhiteSpace.nuspec", + "@type": "PackageEntry", + "compressedLength": 232, + "fullName": "PackageTypeWhiteSpace.nuspec", + "length": 462, + "name": "PackageTypeWhiteSpace.nuspec" + } + ], + "packageTypes": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypewhitespace.0.1.0.json#packagetypes/type1", + "@type": "PackageType", + "name": "type1" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/PackageTypeWhiteSpace.nupkg b/tests/CatalogTests/TestData/PackageTypeWhiteSpace.nupkg new file mode 100644 index 000000000..0039421f2 Binary files /dev/null and b/tests/CatalogTests/TestData/PackageTypeWhiteSpace.nupkg differ diff --git a/tests/CatalogTests/TestData/PackageTypeWhiteSpaceVersion.json b/tests/CatalogTests/TestData/PackageTypeWhiteSpaceVersion.json new file mode 100644 index 000000000..53f240af4 --- /dev/null +++ b/tests/CatalogTests/TestData/PackageTypeWhiteSpaceVersion.json @@ -0,0 +1,89 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/packagetypewhitespaceversion.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "PackageTypeWhiteSpaceVersion", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "PackageTypeWhiteSpaceVersion", + "id": "PackageTypeWhiteSpaceVersion", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "oDnLet1IjgaPxiAHh5Ryp+lIZCd4DW11C8fR5TxuWxaC3t6NcTTLkVnqHvH6E2VHLRf29DVkVYE0fF7PaR/K3Q==", + "packageHashAlgorithm": "SHA512", + "packageSize": 410, + "published": "2017-01-03T08:15:00Z", + "title": "PackageTypeWhiteSpaceVersion", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypewhitespaceversion.0.1.0.json#PackageTypeWhiteSpaceVersion.nuspec", + "@type": "PackageEntry", + "compressedLength": 242, + "fullName": "PackageTypeWhiteSpaceVersion.nuspec", + "length": 506, + "name": "PackageTypeWhiteSpaceVersion.nuspec" + } + ], + "packageTypes": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/packagetypewhitespaceversion.0.1.0.json#packagetypes/type1/1.0.0", + "@type": "PackageType", + "name": "type1", + "version": "1.0.0" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/PackageTypeWhiteSpaceVersion.nupkg b/tests/CatalogTests/TestData/PackageTypeWhiteSpaceVersion.nupkg new file mode 100644 index 000000000..f789a0767 Binary files /dev/null and b/tests/CatalogTests/TestData/PackageTypeWhiteSpaceVersion.nupkg differ diff --git a/tests/CatalogTests/TestData/TestPackage.IconAndIconUrl.0.4.2.nupkg b/tests/CatalogTests/TestData/TestPackage.IconAndIconUrl.0.4.2.nupkg new file mode 100644 index 000000000..844ae7421 Binary files /dev/null and b/tests/CatalogTests/TestData/TestPackage.IconAndIconUrl.0.4.2.nupkg differ diff --git a/tests/CatalogTests/TestData/TestPackage.IconOnlyEmptyType.0.4.2.nupkg b/tests/CatalogTests/TestData/TestPackage.IconOnlyEmptyType.0.4.2.nupkg new file mode 100644 index 000000000..a63036985 Binary files /dev/null and b/tests/CatalogTests/TestData/TestPackage.IconOnlyEmptyType.0.4.2.nupkg differ diff --git a/tests/CatalogTests/TestData/TestPackage.IconOnlyFileType.0.4.2.nupkg b/tests/CatalogTests/TestData/TestPackage.IconOnlyFileType.0.4.2.nupkg new file mode 100644 index 000000000..a29f32f07 Binary files /dev/null and b/tests/CatalogTests/TestData/TestPackage.IconOnlyFileType.0.4.2.nupkg differ diff --git a/tests/CatalogTests/TestData/TestPackage.IconOnlyInvalidType.0.4.2.nupkg b/tests/CatalogTests/TestData/TestPackage.IconOnlyInvalidType.0.4.2.nupkg new file mode 100644 index 000000000..0ea68a398 Binary files /dev/null and b/tests/CatalogTests/TestData/TestPackage.IconOnlyInvalidType.0.4.2.nupkg differ diff --git a/tests/CatalogTests/TestData/TestPackage.IconOnlyNoType.0.4.2.nupkg b/tests/CatalogTests/TestData/TestPackage.IconOnlyNoType.0.4.2.nupkg new file mode 100644 index 000000000..3e47d132b Binary files /dev/null and b/tests/CatalogTests/TestData/TestPackage.IconOnlyNoType.0.4.2.nupkg differ diff --git a/tests/CatalogTests/TestData/TestPackage.LicenseExpression.0.1.0.nupkg b/tests/CatalogTests/TestData/TestPackage.LicenseExpression.0.1.0.nupkg new file mode 100644 index 000000000..fcdaa2180 Binary files /dev/null and b/tests/CatalogTests/TestData/TestPackage.LicenseExpression.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestData/TestPackage.LicenseExpressionAndUrl.0.1.0.nupkg b/tests/CatalogTests/TestData/TestPackage.LicenseExpressionAndUrl.0.1.0.nupkg new file mode 100644 index 000000000..ddf2889c3 Binary files /dev/null and b/tests/CatalogTests/TestData/TestPackage.LicenseExpressionAndUrl.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestData/TestPackage.LicenseFile.0.1.0.nupkg b/tests/CatalogTests/TestData/TestPackage.LicenseFile.0.1.0.nupkg new file mode 100644 index 000000000..3459a952c Binary files /dev/null and b/tests/CatalogTests/TestData/TestPackage.LicenseFile.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestData/TestPackage.LicenseFileAndUrl.0.1.0.nupkg b/tests/CatalogTests/TestData/TestPackage.LicenseFileAndUrl.0.1.0.nupkg new file mode 100644 index 000000000..ed79be49e Binary files /dev/null and b/tests/CatalogTests/TestData/TestPackage.LicenseFileAndUrl.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestData/TestPackage.SemVer2.1.0.0-alpha.1.json b/tests/CatalogTests/TestData/TestPackage.SemVer2.1.0.0-alpha.1.json new file mode 100644 index 000000000..54781005a --- /dev/null +++ b/tests/CatalogTests/TestData/TestPackage.SemVer2.1.0.0-alpha.1.json @@ -0,0 +1,110 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/testpackage.semver2.1.0.0-alpha.1.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "TestPackage.SemVer2", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "Package Description", + "id": "TestPackage.SemVer2", + "isPrerelease": true, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "JMWmWRMlkaAniUmfyFgfNfIQh+EMy1SBzKNbTQ+3xHd+nq2NOCNsVBNcHIKuscAVbE44720YzcL6snUf2RBgXw==", + "packageHashAlgorithm": "SHA512", + "packageSize": 3419, + "published": "2017-01-03T08:15:00Z", + "requireLicenseAcceptance": false, + "verbatimVersion": "1.0.0-alpha.1+githash", + "version": "1.0.0-alpha.1+githash", + "dependencyGroups": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/testpackage.semver2.1.0.0-alpha.1.json#dependencygroup/.netstandard1.4", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/testpackage.semver2.1.0.0-alpha.1.json#dependencygroup/.netstandard1.4/netstandard.library", + "@type": "PackageDependency", + "id": "NETStandard.Library", + "range": "[1.6.1, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/testpackage.semver2.1.0.0-alpha.1.json#dependencygroup/.netstandard1.4/fakepackage.semver2", + "@type": "PackageDependency", + "id": "FakePackage.SemVer2", + "range": "[1.6.1-beta.2, )" + } + ], + "targetFramework": ".NETStandard1.4" + } + ], + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/testpackage.semver2.1.0.0-alpha.1.json#TestPackage.SemVer2.nuspec", + "@type": "PackageEntry", + "compressedLength": 358, + "fullName": "TestPackage.SemVer2.nuspec", + "length": 736, + "name": "TestPackage.SemVer2.nuspec" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/testpackage.semver2.1.0.0-alpha.1.json#lib/netstandard1.4/TestPackage.SemVer2.dll", + "@type": "PackageEntry", + "compressedLength": 1409, + "fullName": "lib/netstandard1.4/TestPackage.SemVer2.dll", + "length": 4096, + "name": "TestPackage.SemVer2.dll" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/TestPackage.SemVer2.1.0.0-alpha.1.nupkg b/tests/CatalogTests/TestData/TestPackage.SemVer2.1.0.0-alpha.1.nupkg new file mode 100644 index 000000000..9bc8e9ccf Binary files /dev/null and b/tests/CatalogTests/TestData/TestPackage.SemVer2.1.0.0-alpha.1.nupkg differ diff --git a/tests/CatalogTests/TestData/WhitespaceDependencyId.0.1.0.json b/tests/CatalogTests/TestData/WhitespaceDependencyId.0.1.0.json new file mode 100644 index 000000000..382de8ad6 --- /dev/null +++ b/tests/CatalogTests/TestData/WhitespaceDependencyId.0.1.0.json @@ -0,0 +1,87 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/whitespacedependencyid.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "WhitespaceDependencyId", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "WhitespaceDependencyId", + "id": "WhitespaceDependencyId", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "2aLzc1k1QHap4vdBIeORBc9vW9Se84orx4/O3TIIMNOGIJ6XDqcWiprTKA89wL4thKB7cwrybr+6T/20dEQ8UQ==", + "packageHashAlgorithm": "SHA512", + "packageSize": 430, + "published": "2017-01-03T08:15:00Z", + "title": "WhitespaceDependencyId", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "dependencyGroups": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/whitespacedependencyid.0.1.0.json#dependencygroup", + "@type": "PackageDependencyGroup" + } + ], + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/whitespacedependencyid.0.1.0.json#WhitespaceDependencyId.nuspec", + "@type": "PackageEntry", + "compressedLength": 238, + "fullName": "WhitespaceDependencyId.nuspec", + "length": 472, + "name": "WhitespaceDependencyId.nuspec" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/WhitespaceDependencyId.0.1.0.nupkg b/tests/CatalogTests/TestData/WhitespaceDependencyId.0.1.0.nupkg new file mode 100644 index 000000000..a76d75913 Binary files /dev/null and b/tests/CatalogTests/TestData/WhitespaceDependencyId.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestData/WhitespaceDependencyVersionRange.0.1.0.json b/tests/CatalogTests/TestData/WhitespaceDependencyVersionRange.0.1.0.json new file mode 100644 index 000000000..8c38d0903 --- /dev/null +++ b/tests/CatalogTests/TestData/WhitespaceDependencyVersionRange.0.1.0.json @@ -0,0 +1,95 @@ +{ + "@id": "http://example/data/2017.01.04.08.15.00/whitespacedependencyversionrange.0.1.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "WhitespaceDependencyVersionRange", + "catalog:commitId": "4aee0ef4-a039-4460-bd5f-98f944e33289", + "catalog:commitTimeStamp": "2017-01-04T08:15:00Z", + "created": "2017-01-01T08:15:00Z", + "description": "WhitespaceDependencyVersionRange", + "id": "WhitespaceDependencyVersionRange", + "isPrerelease": false, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "+a19ygbSmG7q/Ehq8KjBSMf7ToLxNDUcYkkeDQ338dqm9FOsaICc0VemK9smCMcejAMaeWlV0va1u75LHJWBLw==", + "packageHashAlgorithm": "SHA512", + "packageSize": 464, + "published": "2017-01-03T08:15:00Z", + "title": "WhitespaceDependencyVersionRange", + "verbatimVersion": "0.1.0", + "version": "0.1.0", + "dependencyGroups": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/whitespacedependencyversionrange.0.1.0.json#dependencygroup", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/whitespacedependencyversionrange.0.1.0.json#dependencygroup/nuget.versioning", + "@type": "PackageDependency", + "id": "NuGet.Versioning", + "range": "" + } + ] + } + ], + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/whitespacedependencyversionrange.0.1.0.json#WhitespaceDependencyVersionRange.nuspec", + "@type": "PackageEntry", + "compressedLength": 252, + "fullName": "WhitespaceDependencyVersionRange.nuspec", + "length": 530, + "name": "WhitespaceDependencyVersionRange.nuspec" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "vulnerabilities": { + "@id": "vulnerability", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "reasons": { + "@container": "@set" + } + } +} \ No newline at end of file diff --git a/tests/CatalogTests/TestData/WhitespaceDependencyVersionRange.0.1.0.nupkg b/tests/CatalogTests/TestData/WhitespaceDependencyVersionRange.0.1.0.nupkg new file mode 100644 index 000000000..6922bafda Binary files /dev/null and b/tests/CatalogTests/TestData/WhitespaceDependencyVersionRange.0.1.0.nupkg differ diff --git a/tests/CatalogTests/TestHelper.cs b/tests/CatalogTests/TestHelper.cs new file mode 100644 index 000000000..3893ed73e --- /dev/null +++ b/tests/CatalogTests/TestHelper.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.IO; +using Xunit; + +namespace CatalogTests +{ + internal static class TestHelper + { + internal static MemoryStream GetStream(string fileName) + { + var path = Path.GetFullPath(Path.Combine("TestData", fileName)); + + // Multiple tests may try reading the file concurrently. + using (var stream = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + var bytes = new byte[stream.Length]; + + Assert.Equal(bytes.Length, stream.Read(bytes, offset: 0, count: bytes.Length)); + + return new MemoryStream(bytes, index: 0, count: bytes.Length, writable: false); + } + } + } +} \ No newline at end of file diff --git a/tests/Monitoring.PackageLag.Tests/Monitoring.PackageLag.Tests.csproj b/tests/Monitoring.PackageLag.Tests/Monitoring.PackageLag.Tests.csproj index facbc835b..7a058f2c1 100644 --- a/tests/Monitoring.PackageLag.Tests/Monitoring.PackageLag.Tests.csproj +++ b/tests/Monitoring.PackageLag.Tests/Monitoring.PackageLag.Tests.csproj @@ -46,6 +46,10 @@ + + {d44c2e89-2d98-44bd-8712-8ccbe4e67c9c} + NuGet.Protocol.Catalog + {b5147169-e941-4cf8-9fcd-1c123acd3149} Monitoring.PackageLag diff --git a/tests/NgTests/AggregateCursorTests.cs b/tests/NgTests/AggregateCursorTests.cs new file mode 100644 index 000000000..e3dabfa1d --- /dev/null +++ b/tests/NgTests/AggregateCursorTests.cs @@ -0,0 +1,87 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Moq; +using NuGet.Services.Metadata.Catalog; +using Xunit; + +namespace NgTests +{ + public class AggregateCursorTests + { + [Fact] + public void ThrowsIfNoCursors() + { + Assert.Throws(() => new AggregateCursor(null)); + Assert.Throws(() => new AggregateCursor(new List())); + } + + public static IEnumerable ReturnsLeastValue_data + { + get + { + // Single + yield return new object[] + { + new List + { + new DateTime(2017, 4, 13) + } + }; + + // Two with least first + yield return new object[] + { + new List + { + new DateTime(2017, 4, 13), + new DateTime(2017, 4, 14) + } + }; + + // Two with least last + yield return new object[] + { + new List + { + new DateTime(2017, 4, 14), + new DateTime(2017, 4, 13) + } + }; + } + } + + [Theory] + [MemberData(nameof(ReturnsLeastValue_data))] + public async Task ReturnsLeastValue(IEnumerable dates) + { + // Arrange + var cursors = dates.Select(d => CreateReadCursor(d)); + var aggregateCursor = new AggregateCursor(cursors); + + // Act + await aggregateCursor.LoadAsync(CancellationToken.None); + var value = aggregateCursor.Value; + + // Assert + Assert.Equal(dates.Min(), value); + } + + private ReadCursor CreateReadCursor(DateTime date) + { + var cursor = new Mock(); + cursor.Setup(x => x.Value).Returns(() => + { + cursor.Verify(x => x.LoadAsync(It.IsAny())); + return date; + }); + + return cursor.Object; + } + } +} \ No newline at end of file diff --git a/tests/NgTests/AuditRecordHelpersTests.cs b/tests/NgTests/AuditRecordHelpersTests.cs new file mode 100644 index 000000000..f2077e67d --- /dev/null +++ b/tests/NgTests/AuditRecordHelpersTests.cs @@ -0,0 +1,317 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NgTests.Data; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Versioning; +using Xunit; +using Xunit.Abstractions; + +namespace NgTests +{ + public class AuditRecordHelpersTests + { + private readonly ILogger _logger; + + public AuditRecordHelpersTests(ITestOutputHelper testOutputHelper) + { + _logger = new TestLogger(testOutputHelper); + } + + [Fact] + public async Task GetDeletionAuditRecords_HandlesEmptyStorage() + { + var auditingStorage = new MemoryStorage(); + var records = await DeletionAuditEntry.GetAsync(auditingStorage, CancellationToken.None, logger: _logger); + Assert.Empty(records); + } + + [Fact] + public async Task GetDeletionAuditRecords_WithoutFilter() + { + // Arrange + var auditingStorage = new MemoryStorage(); + + var addedAuditEntries = AddDummyAuditRecords(auditingStorage); + + // Act + var auditEntries = await DeletionAuditEntry.GetAsync(auditingStorage, CancellationToken.None, logger: _logger); + + // Assert + Assert.Equal(addedAuditEntries.Item1, auditEntries.Count()); + + for (var i = 0; i < addedAuditEntries.Item1; i++) + { + Assert.Contains(auditEntries, + entry => + entry.PackageId == addedAuditEntries.Item2[i] && + entry.PackageVersion == addedAuditEntries.Item3[i] && + entry.TimestampUtc.HasValue && + entry.TimestampUtc.Value.Ticks == addedAuditEntries.Item4[i].Ticks); + } + } + + [Fact] + public async Task GetDeletionAuditRecords_FiltersOnPackage_EntryExists() + { + // Arrange + var auditingStorage = new MemoryStorage(); + + var targetPackageIdentity = new PackageIdentity("targetPackage", NuGetVersion.Parse("3.2.1")); + + AddAuditRecordToMemoryStorage(auditingStorage, targetPackageIdentity); + + // Act + var auditEntries = await DeletionAuditEntry.GetAsync(CreateStorageFactory(auditingStorage, targetPackageIdentity), CancellationToken.None, targetPackageIdentity, logger: _logger); + + // Assert + Assert.Single(auditEntries); + + var auditEntry = auditEntries.ElementAt(0); + + Assert.Equal(targetPackageIdentity.Id, auditEntry.PackageId); + Assert.Equal(targetPackageIdentity.Version.ToString(), auditEntry.PackageVersion); + } + + [Fact] + public async Task GetDeletionAuditRecords_FiltersOnPackage_EntryMissing() + { + // Arrange + var auditingStorage = new MemoryStorage(); + + var targetPackageIdentity = new PackageIdentity("targetPackage", NuGetVersion.Parse("3.2.1")); + + // Act + var auditEntries = await DeletionAuditEntry.GetAsync(CreateStorageFactory(auditingStorage, targetPackageIdentity), CancellationToken.None, targetPackageIdentity, logger: _logger); + + // Assert + Assert.Empty(auditEntries); + } + + [Fact] + public async Task GetDeletionAuditRecords_FiltersOnMinTimestamp_EntryExists() + { + // Arrange + var auditingStorage = new MemoryStorage(); + + var minTimestamp = DefaultAuditRecordTimeStamp.Add(new TimeSpan(1, 0, 0)); + + AddAuditRecordToMemoryStorage(auditingStorage, package: null, timestamp: minTimestamp); + AddDummyAuditRecords(auditingStorage); + + // Act + var auditEntries = await DeletionAuditEntry.GetAsync(auditingStorage, CancellationToken.None, minTime: minTimestamp, logger: _logger); + + // Assert + Assert.Single(auditEntries); + + var auditEntry = auditEntries.ElementAt(0); + + Assert.True(auditEntry.TimestampUtc.HasValue); + Assert.True(auditEntry.TimestampUtc.Value.Ticks >= minTimestamp.Ticks); + } + + [Fact] + public async Task GetDeletionAuditRecords_FiltersOnMinTimestamp_EntryMissing() + { + // Arrange + var auditingStorage = new MemoryStorage(); + + var minTimestamp = DefaultAuditRecordTimeStamp.Add(new TimeSpan(1, 0, 0)); + + AddDummyAuditRecords(auditingStorage); + + // Act + var auditEntries = await DeletionAuditEntry.GetAsync(auditingStorage, CancellationToken.None, minTime: minTimestamp, logger: _logger); + + // Assert + Assert.Empty(auditEntries); + } + + [Fact] + public async Task GetDeletionAuditRecords_FiltersOnMaxTimestamp_EntryExists() + { + // Arrange + var auditingStorage = new MemoryStorage(); + + var maxTimestamp = DefaultAuditRecordTimeStamp.Subtract(new TimeSpan(1, 0, 0)); + + AddAuditRecordToMemoryStorage(auditingStorage, package: null, timestamp: maxTimestamp); + AddDummyAuditRecords(auditingStorage); + + // Act + var auditEntries = await DeletionAuditEntry.GetAsync(auditingStorage, CancellationToken.None, maxTime: maxTimestamp, logger: _logger); + + // Assert + Assert.Single(auditEntries); + + var auditEntry = auditEntries.ElementAt(0); + + Assert.True(auditEntry.TimestampUtc.HasValue); + Assert.True(auditEntry.TimestampUtc.Value.Ticks <= maxTimestamp.Ticks); + } + + [Fact] + public async Task GetDeletionAuditRecords_FiltersOnMaxTimestamp_EntryMissing() + { + // Arrange + var auditingStorage = new MemoryStorage(); + + var maxTimestamp = DefaultAuditRecordTimeStamp.Subtract(new TimeSpan(1, 0, 0)); + + AddDummyAuditRecords(auditingStorage); + + // Act + var auditEntries = await DeletionAuditEntry.GetAsync(auditingStorage, CancellationToken.None, maxTime: maxTimestamp, logger: _logger); + + // Assert + Assert.Empty(auditEntries); + } + + [Fact] + public async Task GetDeletionAuditRecords_FiltersOnAll_EntryExists() + { + // Arrange + var auditingStorage = new MemoryStorage(); + + var targetPackageIdentity = new PackageIdentity("targetPackage", NuGetVersion.Parse("3.2.1")); + var minTimestamp = DefaultAuditRecordTimeStamp.Subtract(new TimeSpan(2, 0, 0)); + var maxTimestamp = DefaultAuditRecordTimeStamp.Subtract(new TimeSpan(1, 0, 0)); + + AddAuditRecordToMemoryStorage(auditingStorage, targetPackageIdentity, timestamp: minTimestamp.Add(new TimeSpan((maxTimestamp.Ticks - minTimestamp.Ticks) / 2))); + AddDummyAuditRecords(auditingStorage, (count) => targetPackageIdentity.Id, (count) => targetPackageIdentity.Version.ToNormalizedString()); + + // Act + var auditEntries = await DeletionAuditEntry.GetAsync(CreateStorageFactory(auditingStorage, targetPackageIdentity), CancellationToken.None, targetPackageIdentity, minTimestamp, maxTimestamp, logger: _logger); + + // Assert + Assert.Single(auditEntries); + + var auditEntry = auditEntries.ElementAt(0); + + Assert.Equal(targetPackageIdentity.Id, auditEntry.PackageId); + Assert.Equal(targetPackageIdentity.Version.ToString(), auditEntry.PackageVersion); + Assert.True(auditEntry.TimestampUtc.HasValue); + Assert.True(auditEntry.TimestampUtc.Value.Ticks >= minTimestamp.Ticks); + Assert.True(auditEntry.TimestampUtc.Value.Ticks <= maxTimestamp.Ticks); + } + + [Fact] + public async Task GetDeletionAuditRecords_FiltersOnAll_EntryMissing() + { + // Arrange + var auditingStorage = new MemoryStorage(); + + var targetPackageIdentity = new PackageIdentity("targetPackage", NuGetVersion.Parse("3.2.1")); + var minTimestamp = DefaultAuditRecordTimeStamp.Subtract(new TimeSpan(2, 0, 0)); + var maxTimestamp = DefaultAuditRecordTimeStamp.Subtract(new TimeSpan(1, 0, 0)); + + AddDummyAuditRecords(auditingStorage, (count) => targetPackageIdentity.Id, (count) => targetPackageIdentity.Version.ToNormalizedString()); + + // Act + var auditEntries = await DeletionAuditEntry.GetAsync(CreateStorageFactory(auditingStorage, targetPackageIdentity), CancellationToken.None, targetPackageIdentity, minTimestamp, maxTimestamp, logger: _logger); + + // Assert + Assert.Empty(auditEntries); + } + + private DateTime DefaultAuditRecordTimeStamp = DateTime.Parse("2017-03-31T11:57:35Z"); + + private StorageFactory CreateStorageFactory(Storage storage, PackageIdentity targetPackage) + { + var auditingStorageFactoryMock = new Mock(); + + auditingStorageFactoryMock + .Setup(a => a.Create(It.Is(n => n == $"{targetPackage.Id.ToLower()}/{targetPackage.Version.ToNormalizedString().ToLower()}"))) + .Returns(storage); + + return auditingStorageFactoryMock.Object; + } + + private Tuple, List, List> AddDummyAuditRecords(MemoryStorage storage) + { + return AddDummyAuditRecords( + storage, + (count) => $"packageId{count}", + (count) => $"packageVersion{count}"); + } + + private Tuple, List, List> AddDummyAuditRecords( + MemoryStorage storage, + Func getPackageId, + Func getPackageVersion) + { + var packageIds = new List(); + var packageVersions = new List(); + var timestamps = new List(); + var count = 0; + + foreach (var fileSuffix in DeletionAuditEntry.FileNameSuffixes) + { + var packageId = getPackageId(count); + var packageVersion = getPackageVersion(count); + var timestamp = DefaultAuditRecordTimeStamp.Add(new TimeSpan(0, 0, count)); + + AddAuditRecordToMemoryStorage(storage, packageId, packageVersion, timestamp, fileSuffix); + + packageIds.Add(packageId); + packageVersions.Add(packageVersion); + timestamps.Add(timestamp); + count++; + } + + Assert.Equal(count, packageIds.Count()); + Assert.Equal(count, packageVersions.Count()); + Assert.Equal(count, timestamps.Count()); + + return Tuple.Create(count, packageIds, packageVersions, timestamps); + } + + private void AddAuditRecordToMemoryStorage(MemoryStorage storage) + { + AddAuditRecordToMemoryStorage(storage, null, null, null); + } + + private void AddAuditRecordToMemoryStorage(MemoryStorage storage, PackageIdentity package = null, DateTime? timestamp = null, string fileSuffix = null) + { + if (package == null) + { + package = new PackageIdentity("package", NuGetVersion.Parse("1.0.0")); + } + + AddAuditRecordToMemoryStorage(storage, package.Id, package.Version.ToString(), timestamp, fileSuffix); + } + + private void AddAuditRecordToMemoryStorage(MemoryStorage storage, string packageId = "package", string packageVersion = "1.0.0", DateTime? timestamp = null, string fileSuffix = null) + { + var auditTimestamp = timestamp ?? DefaultAuditRecordTimeStamp; + + if (fileSuffix == null) + { + fileSuffix = DeletionAuditEntry.FileNameSuffixes[0]; + } + + var auditRecord = new Uri(storage.BaseAddress, $"package/{packageId}/{packageVersion}/{auditTimestamp.ToFileTimeUtc()}{fileSuffix}"); + storage.Content.TryAdd(auditRecord, MakeDeleteAuditRecord(packageId, packageVersion, auditTimestamp)); + storage.ListMock.TryAdd(auditRecord, new StorageListItem(auditRecord, auditTimestamp)); + } + + private StringStorageContent MakeDeleteAuditRecord(string packageId, string packageVersion, DateTime timestamp) + { + return new StringStorageContent(TestCatalogEntries.DeleteAuditRecordForOtherPackage100 + .Replace("OtherPackage", packageId) + .Replace("1.0.0", packageVersion) + .Replace("2015-01-01T01:01:01.0748028Z", timestamp.ToString())); + } + } +} diff --git a/tests/NgTests/CatalogConstants.cs b/tests/NgTests/CatalogConstants.cs new file mode 100644 index 000000000..2651dce55 --- /dev/null +++ b/tests/NgTests/CatalogConstants.cs @@ -0,0 +1,114 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace NgTests +{ + public static class CatalogConstants + { + public const string AlternatePackage = "alternatePackage"; + public const string AppendOnlyCatalog = "AppendOnlyCatalog"; + public const string Authors = "authors"; + public const string Catalog = "catalog"; + public const string CatalogCatalogPage = "catalog:CatalogPage"; + public const string CatalogCommitId = "catalog:commitId"; + public const string CatalogCommitTimeStamp = "catalog:commitTimeStamp"; + public const string CatalogCount = "catalog:count"; + public const string CatalogDetails = "catalog:details"; + public const string CatalogEntry = "catalogEntry"; + public const string CatalogItem = "catalog:item"; + public const string CatalogPage = "CatalogPage"; + public const string CatalogParent = "catalog:parent"; + public const string CatalogPermalink = "catalog:Permalink"; + public const string CatalogRoot = "CatalogRoot"; + public const string Categories = "categories"; + public const string CommitId = "commitId"; + public const string CommitTimeStamp = "commitTimeStamp"; + public const string CompressedLength = "compressedLength"; + public const string ContainerKeyword = "@container"; + public const string ContextKeyword = "@context"; + public const string Count = "count"; + public const string Created = "created"; + public const string Dependencies = "dependencies"; + public const string Dependency = "dependency"; + public const string DependencyGroup = "dependencyGroup"; + public const string DependencyGroups = "dependencyGroups"; + public const string Deprecation = "deprecation"; + public const string Description = "description"; + public const string Details = "details"; + public const string Entries = "entries"; + public const string FullName = "fullName"; + public const string IconUrl = "iconUrl"; + public const string Id = "id"; + public const string IdKeyword = "@id"; + public const string IsPrerelease = "isPrerelease"; + public const string Item = "item"; + public const string Items = "items"; + public const string Language = "language"; + public const string LastEdited = "lastEdited"; + public const string Length = "length"; + public const string LicenseUrl = "licenseUrl"; + public const string Links = "links"; + public const string Listed = "listed"; + public const string Lower = "lower"; + public const string Message = "message"; + public const string MinClientVersion = "minClientVersion"; + public const string Name = "name"; + public const string NuGet = "nuget"; + public const string NuGetCatalogSchemaPermalinkUri = "http://schema.nuget.org/catalog#Permalink"; + public const string NuGetCatalogSchemaUri = "http://schema.nuget.org/catalog#"; + public const string NuGetId = "nuget:id"; + public const string NuGetLastCreated = "nuget:lastCreated"; + public const string NuGetLastDeleted = "nuget:lastDeleted"; + public const string NuGetLastEdited = "nuget:lastEdited"; + public const string NuGetPackageDelete = "nuget:PackageDelete"; + public const string NuGetPackageDetails = "nuget:PackageDetails"; + public const string NuGetSchemaUri = "http://schema.nuget.org/schema#"; + public const string NuGetVersion = "nuget:version"; + public const string OriginalId = "originalId"; + public const string Package = "Package"; + public const string PackageContent = "packageContent"; + public const string PackageDelete = "PackageDelete"; + public const string PackageDetails = "PackageDetails"; + public const string PackageTypes = "packageTypes"; + public const string PackageType = "PackageType"; + public const string PackageTypeUncapitalized = "packageType"; + public const string PackageEntries = "packageEntries"; + public const string PackageEntry = "PackageEntry"; + public const string PackageEntryUncapitalized = "packageEntry"; + public const string PackageHash = "packageHash"; + public const string PackageHashAlgorithm = "packageHashAlgorithm"; + public const string PackageSize = "packageSize"; + public const string PackageTargetFramework = "packageTargetFramework"; + public const string PackageTargetFrameworks = "packageTargetFrameworks"; + public const string Parent = "parent"; + public const string Permalink = "Permalink"; + public const string ProjectUrl = "projectUrl"; + public const string Published = "published"; + public const string Range = "range"; + public const string Reasons = "reasons"; + public const string RequireLicenseAcceptance = "requireLicenseAcceptance"; + public const string Registration = "registration"; + public const string SetKeyword = "@set"; + public const string SupportedFramework = "supportedFramework"; + public const string SupportedFrameworks = "supportedFrameworks"; + public const string Summary = "summary"; + public const string Tag = "tag"; + public const string Tags = "tags"; + public const string Title = "title"; + public const string TypeKeyword = "@type"; + public const string Upper = "upper"; + public const string VerbatimVersion = "verbatimVersion"; + public const string Version = "version"; + public const string VocabKeyword = "@vocab"; + public const string Vulnerabilities = "vulnerabilities"; + public const string Vulnerability = "vulnerability"; + public const string XmlDateTimeSchemaUri = "http://www.w3.org/2001/XMLSchema#dateTime"; + public const string XmlSchemaUri = "http://www.w3.org/2001/XMLSchema#"; + public const string Xsd = "xsd"; + public const string XsdDateTime = "xsd:dateTime"; + + public const string CommitTimeStampFormat = "yyyy-MM-ddTHH:mm:ss.FFFFFFFZ"; + public const string DateTimeFormat = "yyyy-MM-ddTHH:mm:ss.FFFZ"; + public const string UrlTimeStampFormat = "yyyy.MM.dd.HH.mm.ss"; + } +} \ No newline at end of file diff --git a/tests/NgTests/Data/Catalogs.cs b/tests/NgTests/Data/Catalogs.cs new file mode 100644 index 000000000..695b1d977 --- /dev/null +++ b/tests/NgTests/Data/Catalogs.cs @@ -0,0 +1,208 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NgTests.Infrastructure; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NgTests.Data +{ + public static class Catalogs + { + public static MemoryStorage CreateTestCatalogWithThreePackages() + { + var catalogStorage = new MemoryStorage(); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "index.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesIndex)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "page0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesPage)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/listedpackage.1.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesListedPackage100)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesUnlistedPackage100)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.55/listedpackage.1.0.1.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesListedPackage101)); + + return catalogStorage; + } + + public static MemoryStorage CreateTestCatalogWithCommitThenTwoPackageCommit() + { + var catalogStorage = new MemoryStorage(); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "index.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogWithCommitThenTwoPackageCommitIndex)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "page0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogWithCommitThenTwoPackageCommitPage)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesUnlistedPackage100)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.55/listedpackage.1.0.1.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesListedPackage101)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.55/anotherpackage.1.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogWithCommitThenTwoPackageCommitAnotherPackage100)); + + return catalogStorage; + } + + public static MemoryStorage CreateTestCatalogWithThreePackagesAndDelete() + { + var catalogStorage = new MemoryStorage(); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "index.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesAndDeleteIndex)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "page0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesAndDeletePage)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/listedpackage.1.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesListedPackage100)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesUnlistedPackage100)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.55/listedpackage.1.0.1.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesListedPackage101)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.13.06.40.07/otherpackage.1.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesOtherPackage100)); + + return catalogStorage; + } + + public static MemoryStorage CreateTestCatalogWithPackageCreatedThenDeleted() + { + var catalogStorage = new MemoryStorage(); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "index.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithPackageCreatedThenDeletedIndex)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "page0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithPackageCreatedThenDeletedPage)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/otherpackage.1.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithPackageCreatedThenDeletedPageOtherPackage100Created)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.13.06.40.07/otherpackage.1.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesOtherPackage100)); + + return catalogStorage; + } + + public static MemoryStorage CreateTestCatalogWithThreeItemsForSamePackage(string pageContent) + { + var catalogStorage = new MemoryStorage(); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "index.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesIndex)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "page0.json"), + new StringStorageContent(pageContent)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2017.02.08.16.49.48/mypackage.3.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageMyPackageCreated)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2017.02.08.16.49.59/mypackage.3.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageMyPackageUnlisted)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2017.02.08.17.16.18/mypackage.3.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageMyPackageListed)); + + return catalogStorage; + } + + public static MemoryStorage CreateTestCatalogWithSemVer2Package() + { + var catalogStorage = new MemoryStorage(); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "index.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithSemVer2Index)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "page0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithSemVer2Page000)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/testpackage.semver2.1.0.0-alpha.1.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithSemVer2Package)); + + return catalogStorage; + } + + public static MemoryStorage CreateTestCatalogWithNonNormalizedDelete() + { + var catalogStorage = new MemoryStorage(); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "index.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogWithNonNormalizedDeleteIndex)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "page0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogWithNonNormalizedDeletePage)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/otherpackage.1.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogWithNonNormalizedDeleteOtherPackage100)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.13.06.40.07/otherpackage.1.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogWithNonNormalizedDeleteOtherPackageDelete)); + + return catalogStorage; + } + + public static MemoryStorage CreateTestCatalogWithOnePackage() + { + var catalogStorage = new MemoryStorage(); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "index.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithOnePackageIndex)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "page0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithOnePackagePage)); + + catalogStorage.Content.TryAdd( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/listedpackage.1.0.0.json"), + new StringStorageContent(TestCatalogEntries.TestCatalogStorageWithThreePackagesListedPackage100)); + + return catalogStorage; + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Data/Registrations.cs b/tests/NgTests/Data/Registrations.cs new file mode 100644 index 000000000..5e3bd81e6 --- /dev/null +++ b/tests/NgTests/Data/Registrations.cs @@ -0,0 +1,51 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NgTests.Infrastructure; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NgTests.Data +{ + public static class Registrations + { + public static MemoryStorage CreateTestRegistrations() + { + var registrationStorage = new MemoryStorage(new Uri("https://api.nuget.org/container1/")); + + registrationStorage.Content.TryAdd( + new Uri(registrationStorage.BaseAddress, "cursor.json"), + new StringStorageContent(TestRegistrationEntries.CursorJson)); + + registrationStorage.Content.TryAdd( + new Uri(registrationStorage.BaseAddress, "businessframework/index.json"), + new StringStorageContent(TestRegistrationEntries.BusinessFrameworkIndexJson)); + + registrationStorage.Content.TryAdd( + new Uri(registrationStorage.BaseAddress, "businessframework/0.2.0.json"), + new StringStorageContent(TestRegistrationEntries.BusinessFrameworkVersion1)); + + registrationStorage.Content.TryAdd( + new Uri(registrationStorage.BaseAddress, "automapper/index.json"), + new StringStorageContent(TestRegistrationEntries.AutomapperIndexJson)); + + registrationStorage.Content.TryAdd( + new Uri(registrationStorage.BaseAddress, "automapper/1.1.0.118.json"), + new StringStorageContent(TestRegistrationEntries.AutomapperVersion1)); + + registrationStorage.Content.TryAdd( + new Uri(registrationStorage.BaseAddress, "antlr/index.json"), + new StringStorageContent(TestRegistrationEntries.AntlrIndexJson)); + + registrationStorage.Content.TryAdd( + new Uri(registrationStorage.BaseAddress, "antlr/3.1.1.json"), + new StringStorageContent(TestRegistrationEntries.AntlrVersion1)); + + registrationStorage.Content.TryAdd( + new Uri(registrationStorage.BaseAddress, "antlr/3.1.3.42154.json"), + new StringStorageContent(TestRegistrationEntries.AntlrVersion2)); + + return registrationStorage; + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Data/TestCatalogEntries.Designer.cs b/tests/NgTests/Data/TestCatalogEntries.Designer.cs new file mode 100644 index 000000000..c3b847718 --- /dev/null +++ b/tests/NgTests/Data/TestCatalogEntries.Designer.cs @@ -0,0 +1,757 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace NgTests.Data { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + public class TestCatalogEntries { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal TestCatalogEntries() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + public static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("NgTests.Data.TestCatalogEntries", typeof(TestCatalogEntries).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + public static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to { + /// "record": { + /// "id": "OtherPackage", + /// "version": "1.0.0", + /// "hash": "Cj01MWn4xDwYEcApDHqrR3hCzHv0eDWiOO9TQUA46055TdDBac2FVgITPgtoEERoP5Y1jYSPtfoV/lvpmd4G1Q==", + /// "packageRecord": [ + /// { + /// "key": 1, + /// "packageRegistrationKey": 1, + /// "created": "2015-01-01T01:01:01.827Z", + /// "description": "Package", + /// "downloadCount": 1464, + /// "hashAlgorithm": "SHA512", + /// "hash": "Cj01MWn4xDwYEcApDHqrR3hCzHv0eDWiOO9TQUA46055TdDBac2FVgITPgtoEERoP5Y1jYSPtfoV/lv [rest of string was truncated]";. + /// + public static string DeleteAuditRecordForOtherPackage100 { + get { + return ResourceManager.GetString("DeleteAuditRecordForOtherPackage100", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + /// "commitTimeStamp": "2017-02-08T17:18:55.3335317Z", + /// "count": 3, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/data/2017.02.08.17.16.18/mypackage.3.0.0.json", + /// "@type": "nuget:PackageDetails", + /// "commitId": "0d1d02f5-4800-4c69-96e1-2daaf560edc4", + /// "commitTimeStamp": "2017-02-08T17:16:18.5448099Z", + /// "nuget:id": "mypackage", + /// "nuget:version": [rest of string was truncated]";. + /// + public static string TestCatalogPageForMyPackage_Option1 { + get { + return ResourceManager.GetString("TestCatalogPageForMyPackage_Option1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + /// "commitTimeStamp": "2017-02-08T17:18:55.3335317Z", + /// "count": 3, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/data/2017.02.08.16.49.59/mypackage.3.0.0.json", + /// "@type": "nuget:PackageDetails", + /// "commitId": "d6f4acc0-73c4-4c34-9c92-e1484e749314", + /// "commitTimeStamp": "2017-02-08T16:49:59.6916605Z", + /// "nuget:id": "mypackage", + /// "nuget:version": [rest of string was truncated]";. + /// + public static string TestCatalogPageForMyPackage_Option2 { + get { + return ResourceManager.GetString("TestCatalogPageForMyPackage_Option2", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/data/2017.02.08.16.49.48/mypackage.3.0.0.json", + /// "@type": ["PackageDetails", + /// "catalog:Permalink"], + /// "authors": "NuGet", + /// "catalog:commitId": "51ca9169-a861-4e00-8243-7de49c625da6", + /// "catalog:commitTimeStamp": "2017-02-08T16:49:48.2383888Z", + /// "created": "2017-02-08T16:48:56.567Z", + /// "description": "My package", + /// "id": "mypackage", + /// "isPrerelease": false, + /// "lastEdited": "2017-02-08T16:48:56.567Z", + /// "listed": false, + /// "packageHash": "iGbp+M2f6KbJruB1Y3rUtP [rest of string was truncated]";. + /// + public static string TestCatalogStorageMyPackageCreated { + get { + return ResourceManager.GetString("TestCatalogStorageMyPackageCreated", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/data/2017.02.08.17.16.18/mypackage.3.0.0.json", + /// "@type": ["PackageDetails", + /// "catalog:Permalink"], + /// "authors": "NuGet", + /// "catalog:commitId": "0d1d02f5-4800-4c69-96e1-2daaf560edc4", + /// "catalog:commitTimeStamp": "2017-02-08T17:16:18.5448099Z", + /// "created": "2017-02-08T16:48:56.567Z", + /// "description": "My package", + /// "id": "mypackage", + /// "isPrerelease": false, + /// "lastEdited": "2017-02-08T17:15:40.4Z", + /// "listed": true, + /// "packageHash": "iGbp+M2f6KbJruB1Y3rUtPpZ6 [rest of string was truncated]";. + /// + public static string TestCatalogStorageMyPackageListed { + get { + return ResourceManager.GetString("TestCatalogStorageMyPackageListed", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/data/2017.02.08.16.49.59/mypackage.3.0.0.json", + /// "@type": ["PackageDetails", + /// "catalog:Permalink"], + /// "authors": "NuGet", + /// "catalog:commitId": "d6f4acc0-73c4-4c34-9c92-e1484e749314", + /// "catalog:commitTimeStamp": "2017-02-08T16:49:59.6916605Z", + /// "created": "2017-02-08T16:48:56.567Z", + /// "description": "My package", + /// "id": "mypackage", + /// "isPrerelease": false, + /// "lastEdited": "2017-02-08T16:48:56.567Z", + /// "listed": false, + /// "packageHash": "iGbp+M2f6KbJruB1Y3rUtP [rest of string was truncated]";. + /// + public static string TestCatalogStorageMyPackageUnlisted { + get { + return ResourceManager.GetString("TestCatalogStorageMyPackageUnlisted", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/index.json", + /// "@type": [ + /// "CatalogRoot", + /// "AppendOnlyCatalog", + /// "Permalink" + /// ], + /// "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "count": 1, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "count": 4 + /// } + /// ], + /// "nuget:las [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithOnePackageIndex { + get { + return ResourceManager.GetString("TestCatalogStorageWithOnePackageIndex", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "count": 1, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json", + /// "@type": "nuget:PackageDetails", + /// "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "nuget:id": "ListedPackage", + /// "nuget:ve [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithOnePackagePage { + get { + return ResourceManager.GetString("TestCatalogStorageWithOnePackagePage", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/index.json", + /// "@type": [ + /// "CatalogRoot", + /// "AppendOnlyCatalog", + /// "Permalink" + /// ], + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "count": 1, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "count": 2 + /// } + /// ], + /// "nuget:las [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithPackageCreatedThenDeletedIndex { + get { + return ResourceManager.GetString("TestCatalogStorageWithPackageCreatedThenDeletedIndex", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "count": 2, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.54/otherpackage.1.0.0.json", + /// "@type": "nuget:PackageDetails", + /// "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "nuget:id": "OtherPackage", + /// "nuget:vers [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithPackageCreatedThenDeletedPage { + get { + return ResourceManager.GetString("TestCatalogStorageWithPackageCreatedThenDeletedPage", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.54/otherpackage.1.0.0.json", + /// "@type": [ + /// "PackageDetails", + /// "catalog:Permalink" + /// ], + /// "authors": "NuGet", + /// "catalog:commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "catalog:commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "created": "2015-01-01T00:00:00Z", + /// "description": "Package description.", + /// "id": "ListedPackage", + /// "isPrerelease": false, + /// "lastEdited": "2015-01-01T00:00:00Z", + /// "licenseNames": "", + /// "licenseReport [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithPackageCreatedThenDeletedPageOtherPackage100Created { + get { + return ResourceManager.GetString("TestCatalogStorageWithPackageCreatedThenDeletedPageOtherPackage100Created", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/index.json", + /// "@type": [ + /// "CatalogRoot", + /// "AppendOnlyCatalog", + /// "Permalink" + /// ], + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "count": 1, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "count": 1 + /// } + /// ], + /// "nuget:las [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithSemVer2Index { + get { + return ResourceManager.GetString("TestCatalogStorageWithSemVer2Index", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.54/testpackage.semver2.1.0.0-alpha.1.json", + /// "@type": [ + /// "PackageDetails", + /// "catalog:Permalink" + /// ], + /// "authors": "TestPackage.SemVer2", + /// "catalog:commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "catalog:commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "created": "2017-01-01T08:15:00Z", + /// "description": "Package Description", + /// "id": "TestPackage.SemVer2", + /// "isPrerelease": true, + /// "lastEdited": "2017-01-02T08:15:00Z", + /// "lis [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithSemVer2Package { + get { + return ResourceManager.GetString("TestCatalogStorageWithSemVer2Package", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "count": 1, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.54/testpackage.semver2.1.0.0-alpha.1.json", + /// "@type": "nuget:PackageDetails", + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "nuget:id": "TestPackage.SemVer [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithSemVer2Page000 { + get { + return ResourceManager.GetString("TestCatalogStorageWithSemVer2Page000", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/index.json", + /// "@type": [ + /// "CatalogRoot", + /// "AppendOnlyCatalog", + /// "Permalink" + /// ], + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "count": 1, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "count": 4 + /// } + /// ], + /// "nuget:las [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithThreePackagesAndDeleteIndex { + get { + return ResourceManager.GetString("TestCatalogStorageWithThreePackagesAndDeleteIndex", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "count": 4, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json", + /// "@type": "nuget:PackageDetails", + /// "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "nuget:id": "UnlistedPackage", + /// "nuge [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithThreePackagesAndDeletePage { + get { + return ResourceManager.GetString("TestCatalogStorageWithThreePackagesAndDeletePage", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/index.json", + /// "@type": [ + /// "CatalogRoot", + /// "AppendOnlyCatalog", + /// "Permalink" + /// ], + /// "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + /// "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + /// "count": 1, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + /// "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + /// "count": 3 + /// } + /// ], + /// "nuget:las [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithThreePackagesIndex { + get { + return ResourceManager.GetString("TestCatalogStorageWithThreePackagesIndex", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json", + /// "@type": [ + /// "PackageDetails", + /// "catalog:Permalink" + /// ], + /// "authors": "NuGet", + /// "catalog:commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "catalog:commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "created": "2015-01-01T00:00:00Z", + /// "description": "Package description.", + /// "id": "ListedPackage", + /// "isPrerelease": false, + /// "lastEdited": "2015-01-01T00:00:00Z", + /// "licenseNames": "", + /// "licenseRepor [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithThreePackagesListedPackage100 { + get { + return ResourceManager.GetString("TestCatalogStorageWithThreePackagesListedPackage100", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json", + /// "@type": [ + /// "PackageDetails", + /// "catalog:Permalink" + /// ], + /// "authors": "NuGet", + /// "catalog:commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + /// "catalog:commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + /// "created": "2015-01-01T00:00:00Z", + /// "description": "Package description.", + /// "id": "ListedPackage", + /// "isPrerelease": false, + /// "lastEdited": "2015-01-01T00:00:00Z", + /// "licenseNames": "", + /// "licenseRepor [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithThreePackagesListedPackage101 { + get { + return ResourceManager.GetString("TestCatalogStorageWithThreePackagesListedPackage101", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/data/2015.10.13.06.40.07/otherpackage.1.0.0.json", + /// "@type": [ + /// "PackageDelete", + /// "catalog:Permalink" + /// ], + /// "catalog:commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "catalog:commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "id": "OtherPackage", + /// "originalId": "OtherPackage", + /// "published": "2015-01-01T01:01:01.0748028Z", + /// "version": "1.0.0", + /// "@context": { + /// "@vocab": "http://schema.nuget.org/schema#", + /// "catalog": "http://schema.nuget.org/ [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithThreePackagesOtherPackage100 { + get { + return ResourceManager.GetString("TestCatalogStorageWithThreePackagesOtherPackage100", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + /// "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + /// "count": 3, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json", + /// "@type": "nuget:PackageDetails", + /// "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "nuget:id": "UnlistedPackage", + /// "nuge [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithThreePackagesPage { + get { + return ResourceManager.GetString("TestCatalogStorageWithThreePackagesPage", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json", + /// "@type": [ + /// "PackageDetails", + /// "catalog:Permalink" + /// ], + /// "authors": "NuGet", + /// "catalog:commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "catalog:commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "created": "2015-01-01T00:00:00Z", + /// "description": "Package description.", + /// "id": "UnlistedPackage", + /// "isPrerelease": false, + /// "lastEdited": "2015-01-01T00:00:00Z", + /// "licenseNames": "", + /// "licenseR [rest of string was truncated]";. + /// + public static string TestCatalogStorageWithThreePackagesUnlistedPackage100 { + get { + return ResourceManager.GetString("TestCatalogStorageWithThreePackagesUnlistedPackage100", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.55/anotherpackage.1.0.0.json", + /// "@type": [ + /// "PackageDetails", + /// "catalog:Permalink" + /// ], + /// "authors": "NuGet", + /// "catalog:commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + /// "catalog:commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + /// "created": "2015-01-01T00:00:00Z", + /// "description": "Package description.", + /// "id": "AnotherPackage", + /// "isPrerelease": false, + /// "lastEdited": "2015-01-01T00:00:00Z", + /// "licenseNames": "", + /// "licenseRep [rest of string was truncated]";. + /// + public static string TestCatalogWithCommitThenTwoPackageCommitAnotherPackage100 { + get { + return ResourceManager.GetString("TestCatalogWithCommitThenTwoPackageCommitAnotherPackage100", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/index.json", + /// "@type": [ + /// "CatalogRoot", + /// "AppendOnlyCatalog", + /// "Permalink" + /// ], + /// "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + /// "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + /// "count": 1, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "69345a10-6800-46f2-8131-34c78b0188aa", + /// "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + /// "count": 3 + /// } + /// ], + /// "nuget:las [rest of string was truncated]";. + /// + public static string TestCatalogWithCommitThenTwoPackageCommitIndex { + get { + return ResourceManager.GetString("TestCatalogWithCommitThenTwoPackageCommitIndex", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + /// "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + /// "count": 3, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json", + /// "@type": "nuget:PackageDetails", + /// "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "nuget:id": "UnlistedPackage", + /// "nuge [rest of string was truncated]";. + /// + public static string TestCatalogWithCommitThenTwoPackageCommitPage { + get { + return ResourceManager.GetString("TestCatalogWithCommitThenTwoPackageCommitPage", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/index.json", + /// "@type": [ + /// "CatalogRoot", + /// "AppendOnlyCatalog", + /// "Permalink" + /// ], + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "count": 1, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "count": 2 + /// } + /// ], + /// "nuget:las [rest of string was truncated]";. + /// + public static string TestCatalogWithNonNormalizedDeleteIndex { + get { + return ResourceManager.GetString("TestCatalogWithNonNormalizedDeleteIndex", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.54/otherpackage.1.0.0.json", + /// "@type": [ + /// "PackageDetails", + /// "catalog:Permalink" + /// ], + /// "authors": "NuGet", + /// "catalog:commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "catalog:commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "created": "2015-01-01T00:00:00Z", + /// "description": "Package description.", + /// "id": "OtherPackage", + /// "isPrerelease": false, + /// "lastEdited": "2015-01-01T00:00:00Z", + /// "licenseNames": "", + /// "licenseReportU [rest of string was truncated]";. + /// + public static string TestCatalogWithNonNormalizedDeleteOtherPackage100 { + get { + return ResourceManager.GetString("TestCatalogWithNonNormalizedDeleteOtherPackage100", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/data/2015.10.13.06.40.07/otherpackage.1.0.json", + /// "@type": [ + /// "PackageDelete", + /// "catalog:Permalink" + /// ], + /// "catalog:commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "catalog:commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "id": "OtherPackage", + /// "originalId": "OtherPackage", + /// "published": "2015-01-01T01:01:01.0748028Z", + /// "version": "1.0", + /// "@context": { + /// "@vocab": "http://schema.nuget.org/schema#", + /// "catalog": "http://schema.nuget.org/cata [rest of string was truncated]";. + /// + public static string TestCatalogWithNonNormalizedDeleteOtherPackageDelete { + get { + return ResourceManager.GetString("TestCatalogWithNonNormalizedDeleteOtherPackageDelete", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "http://tempuri.org/page0.json", + /// "@type": "CatalogPage", + /// "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + /// "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + /// "count": 2, + /// "items": [ + /// { + /// "@id": "http://tempuri.org/data/2015.10.12.10.08.54/otherpackage.1.0.0.json", + /// "@type": "nuget:PackageDetails", + /// "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + /// "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + /// "nuget:id": "OtherPackage", + /// "nuget:vers [rest of string was truncated]";. + /// + public static string TestCatalogWithNonNormalizedDeletePage { + get { + return ResourceManager.GetString("TestCatalogWithNonNormalizedDeletePage", resourceCulture); + } + } + } +} diff --git a/tests/NgTests/Data/TestCatalogEntries.resx b/tests/NgTests/Data/TestCatalogEntries.resx new file mode 100644 index 000000000..8cfb5f20e --- /dev/null +++ b/tests/NgTests/Data/TestCatalogEntries.resx @@ -0,0 +1,1791 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + { + "record": { + "id": "OtherPackage", + "version": "1.0.0", + "hash": "Cj01MWn4xDwYEcApDHqrR3hCzHv0eDWiOO9TQUA46055TdDBac2FVgITPgtoEERoP5Y1jYSPtfoV/lvpmd4G1Q==", + "packageRecord": [ + { + "key": 1, + "packageRegistrationKey": 1, + "created": "2015-01-01T01:01:01.827Z", + "description": "Package", + "downloadCount": 1464, + "hashAlgorithm": "SHA512", + "hash": "Cj01MWn4xDwYEcApDHqrR3hCzHv0eDWiOO9TQUA46055TdDBac2FVgITPgtoEERoP5Y1jYSPtfoV/lvpmd4G1Q==", + "isLatest": false, + "lastUpdated": "2015-01-01T01:01:01.957Z", + "licenseUrl": "https://www.nuget.org", + "published": "2015-01-01T01:01:01.827Z", + "packageFileSize": 7838, + "projectUrl": "https://www.nuget.org", + "requiresLicenseAcceptance": false, + "tags": "nuget package", + "version": "1.0.0", + "flattenedAuthors": "NuGet", + "flattenedDependencies": ":", + "isLatestStable": false, + "listed": true, + "isPrerelease": false, + "language": "en-US", + "userKey": 47364, + "hideLicenseReport": false, + "licenseNames": "Apache2", + "licenseReportUrl": "", + "normalizedVersion": "1.0.0" + } + ], + "registrationRecord": [ + { + "key": 1, + "id": "OtherPackage", + "downloadCount": 1636 + } + ], + "reason": "Bulk Delete", + "action": "Deleted" + }, + "actor": { + "machineName": "MACHINE", + "userName": "REDMOND\\user", + "authenticationType": "MachineUser", + "timestampUtc": "2015-01-01T01:01:01.0748028Z" + } +} + + + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + "commitTimeStamp": "2017-02-08T17:18:55.3335317Z", + "count": 3, + "items": [ + { + "@id": "http://tempuri.org/data/2017.02.08.17.16.18/mypackage.3.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "0d1d02f5-4800-4c69-96e1-2daaf560edc4", + "commitTimeStamp": "2017-02-08T17:16:18.5448099Z", + "nuget:id": "mypackage", + "nuget:version": "3.0.0" + }, + { + "@id": "http://tempuri.org/data/2017.02.08.16.49.59/mypackage.3.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "d6f4acc0-73c4-4c34-9c92-e1484e749314", + "commitTimeStamp": "2017-02-08T16:49:59.6916605Z", + "nuget:id": "mypackage", + "nuget:version": "3.0.0" + }, + { + "@id": "http://tempuri.org/data/2017.02.08.16.49.48/mypackage.3.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "51ca9169-a861-4e00-8243-7de49c625da6", + "commitTimeStamp": "2017-02-08T16:49:48.2383888Z", + "nuget:id": "mypackage", + "nuget:version": "3.0.0" + } + ], + "parent": "http://tempuri.org/index.json", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + "commitTimeStamp": "2017-02-08T17:18:55.3335317Z", + "count": 3, + "items": [ + { + "@id": "http://tempuri.org/data/2017.02.08.16.49.59/mypackage.3.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "d6f4acc0-73c4-4c34-9c92-e1484e749314", + "commitTimeStamp": "2017-02-08T16:49:59.6916605Z", + "nuget:id": "mypackage", + "nuget:version": "3.0.0" + }, + { + "@id": "http://tempuri.org/data/2017.02.08.16.49.48/mypackage.3.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "51ca9169-a861-4e00-8243-7de49c625da6", + "commitTimeStamp": "2017-02-08T16:49:48.2383888Z", + "nuget:id": "mypackage", + "nuget:version": "3.0.0" + }, + { + "@id": "http://tempuri.org/data/2017.02.08.17.16.18/mypackage.3.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "0d1d02f5-4800-4c69-96e1-2daaf560edc4", + "commitTimeStamp": "2017-02-08T17:16:18.5448099Z", + "nuget:id": "mypackage", + "nuget:version": "3.0.0" + } + ], + "parent": "http://tempuri.org/index.json", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/data/2017.02.08.16.49.48/mypackage.3.0.0.json", + "@type": ["PackageDetails", + "catalog:Permalink"], + "authors": "NuGet", + "catalog:commitId": "51ca9169-a861-4e00-8243-7de49c625da6", + "catalog:commitTimeStamp": "2017-02-08T16:49:48.2383888Z", + "created": "2017-02-08T16:48:56.567Z", + "description": "My package", + "id": "mypackage", + "isPrerelease": false, + "lastEdited": "2017-02-08T16:48:56.567Z", + "listed": false, + "packageHash": "iGbp+M2f6KbJruB1Y3rUtPpZ6mUsTLT7fvGT1QJcmkD8QNqSZXecWyegeb1kJ7evzRgRpu7gFrN7L3HY2HgZCg==", + "packageHashAlgorithm": "SHA512", + "packageSize": 20652, + "published": "1900-01-01T00:00:00Z", + "requireLicenseAcceptance": false, + "verbatimVersion": "3.0.0", + "version": "3.0.0", + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/data/2017.02.08.17.16.18/mypackage.3.0.0.json", + "@type": ["PackageDetails", + "catalog:Permalink"], + "authors": "NuGet", + "catalog:commitId": "0d1d02f5-4800-4c69-96e1-2daaf560edc4", + "catalog:commitTimeStamp": "2017-02-08T17:16:18.5448099Z", + "created": "2017-02-08T16:48:56.567Z", + "description": "My package", + "id": "mypackage", + "isPrerelease": false, + "lastEdited": "2017-02-08T17:15:40.4Z", + "listed": true, + "packageHash": "iGbp+M2f6KbJruB1Y3rUtPpZ6mUsTLT7fvGT1QJcmkD8QNqSZXecWyegeb1kJ7evzRgRpu7gFrN7L3HY2HgZCg==", + "packageHashAlgorithm": "SHA512", + "packageSize": 20652, + "published": "2017-02-08T16:48:56.567Z", + "requireLicenseAcceptance": false, + "verbatimVersion": "3.0.0", + "version": "3.0.0", + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/data/2017.02.08.16.49.59/mypackage.3.0.0.json", + "@type": ["PackageDetails", + "catalog:Permalink"], + "authors": "NuGet", + "catalog:commitId": "d6f4acc0-73c4-4c34-9c92-e1484e749314", + "catalog:commitTimeStamp": "2017-02-08T16:49:59.6916605Z", + "created": "2017-02-08T16:48:56.567Z", + "description": "My package", + "id": "mypackage", + "isPrerelease": false, + "lastEdited": "2017-02-08T16:48:56.567Z", + "listed": false, + "packageHash": "iGbp+M2f6KbJruB1Y3rUtPpZ6mUsTLT7fvGT1QJcmkD8QNqSZXecWyegeb1kJ7evzRgRpu7gFrN7L3HY2HgZCg==", + "packageHashAlgorithm": "SHA512", + "packageSize": 20652, + "published": "1900-01-01T00:00:00Z", + "requireLicenseAcceptance": false, + "verbatimVersion": "3.0.0", + "version": "3.0.0", + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/index.json", + "@type": [ + "CatalogRoot", + "AppendOnlyCatalog", + "Permalink" + ], + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "count": 1, + "items": [ + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "count": 4 + } + ], + "nuget:lastCreated": "2015-10-12T10:08:54.1506742Z", + "nuget:lastDeleted": "2015-10-12T10:08:54.1506742Z", + "nuget:lastEdited": "2015-10-12T10:08:54.1506742Z", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "count": 1, + "items": [ + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "nuget:id": "ListedPackage", + "nuget:version": "1.0.0" + } + ], + "parent": "http://tempuri.org/index.json", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/index.json", + "@type": [ + "CatalogRoot", + "AppendOnlyCatalog", + "Permalink" + ], + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "count": 1, + "items": [ + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "count": 2 + } + ], + "nuget:lastCreated": "2015-01-01T00:00:00Z", + "nuget:lastDeleted": "2015-01-01T01:01:01.0748028Z", + "nuget:lastEdited": "2015-01-01T01:01:01.0748028Z", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "count": 2, + "items": [ + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/otherpackage.1.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "nuget:id": "OtherPackage", + "nuget:version": "1.0.0" + }, + { + "@id": "http://tempuri.org/data/2015.10.13.06.40.07/otherpackage.1.0.0.json", + "@type": "nuget:PackageDelete", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "nuget:id": "OtherPackage", + "nuget:version": "1.0.0" + } + ], + "parent": "http://tempuri.org/index.json", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/otherpackage.1.0.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "NuGet", + "catalog:commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "catalog:commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "created": "2015-01-01T00:00:00Z", + "description": "Package description.", + "id": "ListedPackage", + "isPrerelease": false, + "lastEdited": "2015-01-01T00:00:00Z", + "licenseNames": "", + "licenseReportUrl": "", + "listed": true, + "packageHash": "EpTkeONwnhX59JBzl5QfuFNNgZADaAbwYxNGn0KEkJ4ylukUQFcS15vISqFUnrWy/+yylox6L6QT3MD/+Us+vg==", + "packageHashAlgorithm": "SHA512", + "packageSize": 2529, + "published": "2015-01-01T00:00:00Z", + "requireLicenseAcceptance": false, + "verbatimVersion": "1.0.0", + "version": "1.0.0", + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/index.json", + "@type": [ + "CatalogRoot", + "AppendOnlyCatalog", + "Permalink" + ], + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "count": 1, + "items": [ + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "count": 1 + } + ], + "nuget:lastCreated": "2015-01-01T00:00:00Z", + "nuget:lastDeleted": "2015-01-01T01:01:01.0748028Z", + "nuget:lastEdited": "2015-01-01T01:01:01.0748028Z", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/testpackage.semver2.1.0.0-alpha.1.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "TestPackage.SemVer2", + "catalog:commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "catalog:commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "created": "2017-01-01T08:15:00Z", + "description": "Package Description", + "id": "TestPackage.SemVer2", + "isPrerelease": true, + "lastEdited": "2017-01-02T08:15:00Z", + "listed": true, + "packageHash": "JMWmWRMlkaAniUmfyFgfNfIQh+EMy1SBzKNbTQ+3xHd+nq2NOCNsVBNcHIKuscAVbE44720YzcL6snUf2RBgXw==", + "packageHashAlgorithm": "SHA512", + "packageSize": 3419, + "published": "2017-01-03T08:15:00Z", + "requireLicenseAcceptance": false, + "verbatimVersion": "1.0.0-alpha.1+githash", + "version": "1.0.0-alpha.1+githash", + "dependencyGroups": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/testpackage.semver2.1.0.0-alpha.1.json#dependencygroup/.netstandard1.4", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/testpackage.semver2.1.0.0-alpha.1.json#dependencygroup/.netstandard1.4/netstandard.library", + "@type": "PackageDependency", + "id": "NETStandard.Library", + "range": "[1.6.1, )" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/testpackage.semver2.1.0.0-alpha.1.json#dependencygroup/.netstandard1.4/fakepackage.semver2", + "@type": "PackageDependency", + "id": "FakePackage.SemVer2", + "range": "[1.6.1-beta.2, )" + } + ], + "targetFramework": ".NETStandard1.4" + } + ], + "packageEntries": [ + { + "@id": "http://example/data/2017.01.04.08.15.00/testpackage.semver2.1.0.0-alpha.1.json#TestPackage.SemVer2.nuspec", + "@type": "PackageEntry", + "compressedLength": 358, + "fullName": "TestPackage.SemVer2.nuspec", + "length": 736, + "name": "TestPackage.SemVer2.nuspec" + }, + { + "@id": "http://example/data/2017.01.04.08.15.00/testpackage.semver2.1.0.0-alpha.1.json#lib/netstandard1.4/TestPackage.SemVer2.dll", + "@type": "PackageEntry", + "compressedLength": 1409, + "fullName": "lib/netstandard1.4/TestPackage.SemVer2.dll", + "length": 4096, + "name": "TestPackage.SemVer2.dll" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "count": 1, + "items": [ + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/testpackage.semver2.1.0.0-alpha.1.json", + "@type": "nuget:PackageDetails", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "nuget:id": "TestPackage.SemVer2", + "nuget:version": "1.0.0-alpha.1+githash" + } + ], + "parent": "http://tempuri.org/index.json", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/index.json", + "@type": [ + "CatalogRoot", + "AppendOnlyCatalog", + "Permalink" + ], + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "count": 1, + "items": [ + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "count": 4 + } + ], + "nuget:lastCreated": "2015-01-01T00:00:00Z", + "nuget:lastDeleted": "2015-01-01T01:01:01.0748028Z", + "nuget:lastEdited": "2015-01-01T01:01:01.0748028Z", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "count": 4, + "items": [ + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "nuget:id": "UnlistedPackage", + "nuget:version": "1.0.0" + }, + { + "@id": "http://tempuri.org/data/2015.10.13.06.40.07/otherpackage.1.0.0.json", + "@type": "nuget:PackageDelete", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "nuget:id": "OtherPackage", + "nuget:version": "1.0.0" + }, + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json", + "@type": "nuget:PackageDetails", + "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + "nuget:id": "ListedPackage", + "nuget:version": "1.0.1" + }, + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "nuget:id": "ListedPackage", + "nuget:version": "1.0.0" + } + ], + "parent": "http://tempuri.org/index.json", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/index.json", + "@type": [ + "CatalogRoot", + "AppendOnlyCatalog", + "Permalink" + ], + "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + "count": 1, + "items": [ + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + "count": 3 + } + ], + "nuget:lastCreated": "2015-01-01T00:00:00Z", + "nuget:lastDeleted": "2015-01-01T00:00:00Z", + "nuget:lastEdited": "2015-01-01T00:00:00Z", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "NuGet", + "catalog:commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "catalog:commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "created": "2015-01-01T00:00:00Z", + "description": "Package description.", + "id": "ListedPackage", + "isPrerelease": false, + "lastEdited": "2015-01-01T00:00:00Z", + "licenseNames": "", + "licenseReportUrl": "", + "listed": true, + "packageHash": "EpTkeONwnhX59JBzl5QfuFNNgZADaAbwYxNGn0KEkJ4ylukUQFcS15vISqFUnrWy/+yylox6L6QT3MD/+Us+vg==", + "packageHashAlgorithm": "SHA512", + "packageSize": 2529, + "published": "2015-01-01T00:00:00Z", + "requireLicenseAcceptance": false, + "verbatimVersion": "1.0.0", + "version": "1.0.0", + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "NuGet", + "catalog:commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + "catalog:commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + "created": "2015-01-01T00:00:00Z", + "description": "Package description.", + "id": "ListedPackage", + "isPrerelease": false, + "lastEdited": "2015-01-01T00:00:00Z", + "licenseNames": "", + "licenseReportUrl": "", + "listed": true, + "packageHash": "GU/C5uGF1GQtHQpjI1e3dsnpelbRcBE+EGDUrrtWvRvucI31XXZQltvd/CY/rK9Fm9lTwMxGMVrUgRLz248FmQ==", + "packageHashAlgorithm": "SHA512", + "packageSize": 2529, + "published": "2015-01-01T00:00:00Z", + "requireLicenseAcceptance": false, + "verbatimVersion": "1.0.1", + "version": "1.0.1", + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/data/2015.10.13.06.40.07/otherpackage.1.0.0.json", + "@type": [ + "PackageDelete", + "catalog:Permalink" + ], + "catalog:commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "catalog:commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "id": "OtherPackage", + "originalId": "OtherPackage", + "published": "2015-01-01T01:01:01.0748028Z", + "version": "1.0.0", + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "details": "catalog:details", + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "published": { + "@type": "xsd:dateTime" + }, + "categories": { + "@container": "@set" + }, + "entries": { + "@container": "@set" + }, + "links": { + "@container": "@set" + }, + "tags": { + "@container": "@set" + }, + "packageContent": { + "@type": "@id" + } + } +} + + + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + "count": 3, + "items": [ + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "nuget:id": "UnlistedPackage", + "nuget:version": "1.0.0" + }, + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "nuget:id": "ListedPackage", + "nuget:version": "1.0.0" + }, + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json", + "@type": "nuget:PackageDetails", + "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + "nuget:id": "ListedPackage", + "nuget:version": "1.0.1" + } + ], + "parent": "http://tempuri.org/index.json", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "NuGet", + "catalog:commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "catalog:commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "created": "2015-01-01T00:00:00Z", + "description": "Package description.", + "id": "UnlistedPackage", + "isPrerelease": false, + "lastEdited": "2015-01-01T00:00:00Z", + "licenseNames": "", + "licenseReportUrl": "", + "listed": false, + "packageHash": "0LUKl3HqBsrLxjTaA+J6vaa+XB5zWb5hTJb1Ht74oIzPsBJmZalApZAIdueMgzbsUxG74jjfnDZWoxUrs2A3dg==", + "packageHashAlgorithm": "SHA512", + "packageSize": 2540, + "published": "1900-01-01T01:00:00+00:00", + "requireLicenseAcceptance": false, + "verbatimVersion": "1.0.0", + "version": "1.0.0", + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.55/anotherpackage.1.0.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "NuGet", + "catalog:commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + "catalog:commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + "created": "2015-01-01T00:00:00Z", + "description": "Package description.", + "id": "AnotherPackage", + "isPrerelease": false, + "lastEdited": "2015-01-01T00:00:00Z", + "licenseNames": "", + "licenseReportUrl": "", + "listed": true, + "packageHash": "EpTkeONwnhX59JBzl5QfuFNNgZADaAbwYxNGn0KEkJ4ylukUQFcS15vISqFUnrWy/+yylox6L6QT3MD/+Us+vg==", + "packageHashAlgorithm": "SHA512", + "packageSize": 2529, + "published": "2015-01-01T00:00:00Z", + "requireLicenseAcceptance": false, + "verbatimVersion": "1.0.0", + "version": "1.0.0", + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/index.json", + "@type": [ + "CatalogRoot", + "AppendOnlyCatalog", + "Permalink" + ], + "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + "count": 1, + "items": [ + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "69345a10-6800-46f2-8131-34c78b0188aa", + "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + "count": 3 + } + ], + "nuget:lastCreated": "2015-01-01T00:00:00Z", + "nuget:lastDeleted": "2015-01-01T00:00:00Z", + "nuget:lastEdited": "2015-01-01T00:00:00Z", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + "count": 3, + "items": [ + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "nuget:id": "UnlistedPackage", + "nuget:version": "1.0.0" + }, + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json", + "@type": "nuget:PackageDetails", + "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + "nuget:id": "ListedPackage", + "nuget:version": "1.0.1" + }, + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.55/anotherpackage.1.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + "commitTimeStamp": "2015-10-12T10:08:55.3335317Z", + "nuget:id": "AnotherPackage", + "nuget:version": "1.0.0" + } + ], + "parent": "http://tempuri.org/index.json", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/index.json", + "@type": [ + "CatalogRoot", + "AppendOnlyCatalog", + "Permalink" + ], + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "count": 1, + "items": [ + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "count": 2 + } + ], + "nuget:lastCreated": "2015-01-01T00:00:00Z", + "nuget:lastDeleted": "2015-01-01T01:01:01.0748028Z", + "nuget:lastEdited": "2015-01-01T01:01:01.0748028Z", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/otherpackage.1.0.0.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "NuGet", + "catalog:commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "catalog:commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "created": "2015-01-01T00:00:00Z", + "description": "Package description.", + "id": "OtherPackage", + "isPrerelease": false, + "lastEdited": "2015-01-01T00:00:00Z", + "licenseNames": "", + "licenseReportUrl": "", + "listed": true, + "packageHash": "EpTkeONwnhX59JBzl5QfuFNNgZADaAbwYxNGn0KEkJ4ylukUQFcS15vISqFUnrWy/+yylox6L6QT3MD/+Us+vg==", + "packageHashAlgorithm": "SHA512", + "packageSize": 2529, + "published": "2015-01-01T00:00:00Z", + "requireLicenseAcceptance": false, + "verbatimVersion": "1.0.0", + "version": "1.0.0", + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + } + } +} + + + { + "@id": "http://tempuri.org/data/2015.10.13.06.40.07/otherpackage.1.0.json", + "@type": [ + "PackageDelete", + "catalog:Permalink" + ], + "catalog:commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "catalog:commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "id": "OtherPackage", + "originalId": "OtherPackage", + "published": "2015-01-01T01:01:01.0748028Z", + "version": "1.0", + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "details": "catalog:details", + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "published": { + "@type": "xsd:dateTime" + }, + "categories": { + "@container": "@set" + }, + "entries": { + "@container": "@set" + }, + "links": { + "@container": "@set" + }, + "tags": { + "@container": "@set" + }, + "packageContent": { + "@type": "@id" + } + } +} + + + { + "@id": "http://tempuri.org/page0.json", + "@type": "CatalogPage", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "count": 2, + "items": [ + { + "@id": "http://tempuri.org/data/2015.10.12.10.08.54/otherpackage.1.0.0.json", + "@type": "nuget:PackageDetails", + "commitId": "9a37734f-1960-4c07-8934-c8bc797e35c1", + "commitTimeStamp": "2015-10-12T10:08:54.1506742Z", + "nuget:id": "OtherPackage", + "nuget:version": "1.0.0" + }, + { + "@id": "http://tempuri.org/data/2015.10.13.06.40.07/otherpackage.1.0.json", + "@type": "nuget:PackageDelete", + "commitId": "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + "commitTimeStamp": "2015-10-13T06:40:07.7850657Z", + "nuget:id": "OtherPackage", + "nuget:version": "1.0" + } + ], + "parent": "http://tempuri.org/index.json", + "@context": { + "@vocab": "http://schema.nuget.org/catalog#", + "nuget": "http://schema.nuget.org/schema#", + "items": { + "@id": "item", + "@container": "@set" + }, + "parent": { + "@type": "@id" + }, + "commitTimeStamp": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastCreated": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastEdited": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + }, + "nuget:lastDeleted": { + "@type": "http://www.w3.org/2001/XMLSchema#dateTime" + } + } +} + + \ No newline at end of file diff --git a/tests/NgTests/Data/TestRegistrationEntries.Designer.cs b/tests/NgTests/Data/TestRegistrationEntries.Designer.cs new file mode 100644 index 000000000..d2700538b --- /dev/null +++ b/tests/NgTests/Data/TestRegistrationEntries.Designer.cs @@ -0,0 +1,137 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace NgTests.Data { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "15.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + public class TestRegistrationEntries { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal TestRegistrationEntries() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + public static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("NgTests.Data.TestRegistrationEntries", typeof(TestRegistrationEntries).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + public static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/container1/antlr/index.json","@type":["catalog:CatalogRoot","PackageRegistration","catalog:Permalink"],"commitId":"682b9969-ac84-45ca-860d-08bc3d9fa0f1","commitTimeStamp":"2016-02-26T08:28:24.6757212Z","count":1,"items":[{"@id":"https://api.nuget.org/container1/antlr/index.json#page/3.1.1/3.1.3.42154","@type":"catalog:CatalogPage","commitId":"682b9969-ac84-45ca-860d-08bc3d9fa0f1","commitTimeStamp":"2016-02-26T08:28:24.6757212Z","count":2,"items":[{"@id":"https://api.nuget.org/c [rest of string was truncated]";. + /// + public static string AntlrIndexJson { + get { + return ResourceManager.GetString("AntlrIndexJson", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/container1/antlr/3.1.1.json","@type":["Package","http://schema.nuget.org/catalog#Permalink"],"catalogEntry":"https://api.nuget.org/v3/catalog0-v2v3/data/2016.01.05.09.32.40/antlr.3.1.1.json","listed":true,"packageContent":"https://api.nuget.org/packages/antlr.3.1.1.nupkg","published":"2011-01-07T08:49:52.917+01:00","registration":"https://api.nuget.org/container1/antlr/index.json","@context":{"@vocab":"http://schema.nuget.org/schema#","xsd":"http://www.w3.org/2001/XMLSchema#"," [rest of string was truncated]";. + /// + public static string AntlrVersion1 { + get { + return ResourceManager.GetString("AntlrVersion1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/container1/antlr/3.1.3.42154.json","@type":["Package","http://schema.nuget.org/catalog#Permalink"],"catalogEntry":"https://api.nuget.org/v3/catalog0-v2v3/data/2016.01.05.09.32.40/antlr.3.1.3.42154.json","listed":true,"packageContent":"https://api.nuget.org/packages/antlr.3.1.3.42154.nupkg","published":"2011-01-07T08:49:54.527+01:00","registration":"https://api.nuget.org/container1/antlr/index.json","@context":{"@vocab":"http://schema.nuget.org/schema#","xsd":"http://www.w3.org/ [rest of string was truncated]";. + /// + public static string AntlrVersion2 { + get { + return ResourceManager.GetString("AntlrVersion2", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/container1/automapper/index.json","@type":["catalog:CatalogRoot","PackageRegistration","catalog:Permalink"],"commitId":"e1746e1d-fdc7-4aab-bee5-09fb677026fc","commitTimeStamp":"2016-02-26T08:28:33.5999394Z","count":1,"items":[{"@id":"https://api.nuget.org/container1/automapper/index.json#page/1.1.0.118/1.1.0.118","@type":"catalog:CatalogPage","commitId":"e1746e1d-fdc7-4aab-bee5-09fb677026fc","commitTimeStamp":"2016-02-26T08:28:33.5999394Z","count":1,"items":[{"@id":"https://api [rest of string was truncated]";. + /// + public static string AutomapperIndexJson { + get { + return ResourceManager.GetString("AutomapperIndexJson", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/container1/automapper/1.1.0.118.json","@type":["Package","http://schema.nuget.org/catalog#Permalink"],"catalogEntry":"https://api.nuget.org/v3/catalog0-v2v3/data/2016.01.05.09.32.46/automapper.1.1.0.118.json","listed":true,"packageContent":"https://api.nuget.org/packages/automapper.1.1.0.118.nupkg","published":"2011-09-07T22:48:17.403+02:00","registration":"https://api.nuget.org/container1/automapper/index.json","@context":{"@vocab":"http://schema.nuget.org/schema#","xsd":"http [rest of string was truncated]";. + /// + public static string AutomapperVersion1 { + get { + return ResourceManager.GetString("AutomapperVersion1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/container1/businessframework/index.json","@type":["catalog:CatalogRoot","PackageRegistration","catalog:Permalink"],"commitId":"2df1945a-8dd4-4b44-8330-527f4204f370","commitTimeStamp":"2016-02-26T09:22:09.4438705Z","count":1,"items":[{"@id":"https://api.nuget.org/container1/businessframework/index.json#page/0.2.0/0.2.0","@type":"catalog:CatalogPage","commitId":"2df1945a-8dd4-4b44-8330-527f4204f370","commitTimeStamp":"2016-02-26T09:22:09.4438705Z","count":1,"items":[{"@id":"https [rest of string was truncated]";. + /// + public static string BusinessFrameworkIndexJson { + get { + return ResourceManager.GetString("BusinessFrameworkIndexJson", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/container1/businessframework/0.2.0.json","@type":["Package","http://schema.nuget.org/catalog#Permalink"],"catalogEntry":"https://api.nuget.org/v3/catalog0-v2v3/data/2016.01.05.09.32.46/businessframework.0.2.0.json","listed":true,"packageContent":"https://api.nuget.org/packages/businessframework.0.2.0.nupkg","published":"2011-01-07T08:50:40.807+01:00","registration":"https://api.nuget.org/container1/businessframework/index.json","@context":{"@vocab":"http://schema.nuget.org/sche [rest of string was truncated]";. + /// + public static string BusinessFrameworkVersion1 { + get { + return ResourceManager.GetString("BusinessFrameworkVersion1", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "value": "2016-01-05T09:32:51.9339612" + ///}. + /// + public static string CursorJson { + get { + return ResourceManager.GetString("CursorJson", resourceCulture); + } + } + } +} diff --git a/tests/NgTests/Data/TestRegistrationEntries.resx b/tests/NgTests/Data/TestRegistrationEntries.resx new file mode 100644 index 000000000..a01ff4558 --- /dev/null +++ b/tests/NgTests/Data/TestRegistrationEntries.resx @@ -0,0 +1,146 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + {"@id":"https://api.nuget.org/container1/antlr/index.json","@type":["catalog:CatalogRoot","PackageRegistration","catalog:Permalink"],"commitId":"682b9969-ac84-45ca-860d-08bc3d9fa0f1","commitTimeStamp":"2016-02-26T08:28:24.6757212Z","count":1,"items":[{"@id":"https://api.nuget.org/container1/antlr/index.json#page/3.1.1/3.1.3.42154","@type":"catalog:CatalogPage","commitId":"682b9969-ac84-45ca-860d-08bc3d9fa0f1","commitTimeStamp":"2016-02-26T08:28:24.6757212Z","count":2,"items":[{"@id":"https://api.nuget.org/container1/antlr/3.1.1.json","@type":"Package","commitId":"682b9969-ac84-45ca-860d-08bc3d9fa0f1","commitTimeStamp":"2016-02-26T08:28:24.6757212Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0-v2v3/data/2016.01.05.09.32.40/antlr.3.1.1.json","@type":"PackageDetails","authors":"Terence Parr","description":"ANother Tool for Language Recognition, is a language tool that provides a framework for constructing recognizers, interpreters, compilers, and translators from grammatical descriptions containing actions in a variety of target languages.","iconUrl":"","id":"Antlr","language":"en-US","licenseUrl":"","listed":true,"minClientVersion":"","packageContent":"https://api.nuget.org/packages/antlr.3.1.1.nupkg","projectUrl":"","published":"2011-01-07T08:49:52.917+01:00","requireLicenseAcceptance":false,"summary":"ANother Tool for Language Recognition, is a language tool that provides a framework for constructing recognizers, interpreters, compilers, and translators from grammatical descriptions containing actions in a variety of target languages.","tags":[""],"title":"","version":"3.1.1"},"packageContent":"https://api.nuget.org/packages/antlr.3.1.1.nupkg","registration":"https://api.nuget.org/container1/antlr/index.json"},{"@id":"https://api.nuget.org/container1/antlr/3.1.3.42154.json","@type":"Package","commitId":"682b9969-ac84-45ca-860d-08bc3d9fa0f1","commitTimeStamp":"2016-02-26T08:28:24.6757212Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0-v2v3/data/2016.01.05.09.32.40/antlr.3.1.3.42154.json","@type":"PackageDetails","authors":"Terence Parr","description":"ANother Tool for Language Recognition, is a language tool that provides a framework for constructing recognizers, interpreters, compilers, and translators from grammatical descriptions containing actions in a variety of target languages.","iconUrl":"","id":"Antlr","language":"en-US","licenseUrl":"","listed":true,"minClientVersion":"","packageContent":"https://api.nuget.org/packages/antlr.3.1.3.42154.nupkg","projectUrl":"","published":"2011-01-07T08:49:54.527+01:00","requireLicenseAcceptance":false,"summary":"ANother Tool for Language Recognition, is a language tool that provides a framework for constructing recognizers, interpreters, compilers, and translators from grammatical descriptions containing actions in a variety of target languages.","tags":[""],"title":"","version":"3.1.3.42154"},"packageContent":"https://api.nuget.org/packages/antlr.3.1.3.42154.nupkg","registration":"https://api.nuget.org/container1/antlr/index.json"}],"parent":"https://api.nuget.org/container1/antlr/index.json","lower":"3.1.1","upper":"3.1.3.42154"}],"@context":{"@vocab":"http://schema.nuget.org/schema#","catalog":"http://schema.nuget.org/catalog#","xsd":"http://www.w3.org/2001/XMLSchema#","items":{"@id":"catalog:item","@container":"@set"},"commitTimeStamp":{"@id":"catalog:commitTimeStamp","@type":"xsd:dateTime"},"commitId":{"@id":"catalog:commitId"},"count":{"@id":"catalog:count"},"parent":{"@id":"catalog:parent","@type":"@id"},"tags":{"@container":"@set","@id":"tag"},"packageTargetFrameworks":{"@container":"@set","@id":"packageTargetFramework"},"dependencyGroups":{"@container":"@set","@id":"dependencyGroup"},"dependencies":{"@container":"@set","@id":"dependency"},"packageContent":{"@type":"@id"},"published":{"@type":"xsd:dateTime"},"registration":{"@type":"@id"}}} + + + {"@id":"https://api.nuget.org/container1/antlr/3.1.1.json","@type":["Package","http://schema.nuget.org/catalog#Permalink"],"catalogEntry":"https://api.nuget.org/v3/catalog0-v2v3/data/2016.01.05.09.32.40/antlr.3.1.1.json","listed":true,"packageContent":"https://api.nuget.org/packages/antlr.3.1.1.nupkg","published":"2011-01-07T08:49:52.917+01:00","registration":"https://api.nuget.org/container1/antlr/index.json","@context":{"@vocab":"http://schema.nuget.org/schema#","xsd":"http://www.w3.org/2001/XMLSchema#","catalogEntry":{"@type":"@id"},"registration":{"@type":"@id"},"packageContent":{"@type":"@id"},"published":{"@type":"xsd:dateTime"}}} + + + {"@id":"https://api.nuget.org/container1/antlr/3.1.3.42154.json","@type":["Package","http://schema.nuget.org/catalog#Permalink"],"catalogEntry":"https://api.nuget.org/v3/catalog0-v2v3/data/2016.01.05.09.32.40/antlr.3.1.3.42154.json","listed":true,"packageContent":"https://api.nuget.org/packages/antlr.3.1.3.42154.nupkg","published":"2011-01-07T08:49:54.527+01:00","registration":"https://api.nuget.org/container1/antlr/index.json","@context":{"@vocab":"http://schema.nuget.org/schema#","xsd":"http://www.w3.org/2001/XMLSchema#","catalogEntry":{"@type":"@id"},"registration":{"@type":"@id"},"packageContent":{"@type":"@id"},"published":{"@type":"xsd:dateTime"}}} + + + {"@id":"https://api.nuget.org/container1/automapper/index.json","@type":["catalog:CatalogRoot","PackageRegistration","catalog:Permalink"],"commitId":"e1746e1d-fdc7-4aab-bee5-09fb677026fc","commitTimeStamp":"2016-02-26T08:28:33.5999394Z","count":1,"items":[{"@id":"https://api.nuget.org/container1/automapper/index.json#page/1.1.0.118/1.1.0.118","@type":"catalog:CatalogPage","commitId":"e1746e1d-fdc7-4aab-bee5-09fb677026fc","commitTimeStamp":"2016-02-26T08:28:33.5999394Z","count":1,"items":[{"@id":"https://api.nuget.org/container1/automapper/1.1.0.118.json","@type":"Package","commitId":"e1746e1d-fdc7-4aab-bee5-09fb677026fc","commitTimeStamp":"2016-02-26T08:28:33.5999394Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0-v2v3/data/2016.01.05.09.32.46/automapper.1.1.0.118.json","@type":"PackageDetails","authors":"Headspring,Jimmy Bogard","description":"A convention-based object-object mapper. AutoMapper uses a fluent configuration API to define an object-object mapping strategy. AutoMapper uses a convention-based matching algorithm to match up source to destination values. Currently, AutoMapper is geared towards model projection scenarios to flatten complex object models to DTOs and other simple objects, whose design is better suited for serialization, communication, messaging, or simply an anti-corruption layer between the domain and application layer.","iconUrl":"","id":"AutoMapper","language":"en-US","licenseUrl":"","listed":true,"minClientVersion":"","packageContent":"https://api.nuget.org/packages/automapper.1.1.0.118.nupkg","projectUrl":"","published":"2011-09-07T22:48:17.403+02:00","requireLicenseAcceptance":false,"summary":"A convention-based object-object mapper. AutoMapper uses a fluent configuration API to define an object-object mapping strategy. AutoMapper uses a convention-based matching algorithm to match up source to destination values. Currently, AutoMapper is geared towards model projection scenarios to flatten complex object models to DTOs and other simple objects, whose design is better suited for serialization, communication, messaging, or simply an anti-corruption layer between the domain and application layer.","tags":[""],"title":"","version":"1.1.0.118"},"packageContent":"https://api.nuget.org/packages/automapper.1.1.0.118.nupkg","registration":"https://api.nuget.org/container1/automapper/index.json"}],"parent":"https://api.nuget.org/container1/automapper/index.json","lower":"1.1.0.118","upper":"1.1.0.118"}],"@context":{"@vocab":"http://schema.nuget.org/schema#","catalog":"http://schema.nuget.org/catalog#","xsd":"http://www.w3.org/2001/XMLSchema#","items":{"@id":"catalog:item","@container":"@set"},"commitTimeStamp":{"@id":"catalog:commitTimeStamp","@type":"xsd:dateTime"},"commitId":{"@id":"catalog:commitId"},"count":{"@id":"catalog:count"},"parent":{"@id":"catalog:parent","@type":"@id"},"tags":{"@container":"@set","@id":"tag"},"packageTargetFrameworks":{"@container":"@set","@id":"packageTargetFramework"},"dependencyGroups":{"@container":"@set","@id":"dependencyGroup"},"dependencies":{"@container":"@set","@id":"dependency"},"packageContent":{"@type":"@id"},"published":{"@type":"xsd:dateTime"},"registration":{"@type":"@id"}}} + + + {"@id":"https://api.nuget.org/container1/automapper/1.1.0.118.json","@type":["Package","http://schema.nuget.org/catalog#Permalink"],"catalogEntry":"https://api.nuget.org/v3/catalog0-v2v3/data/2016.01.05.09.32.46/automapper.1.1.0.118.json","listed":true,"packageContent":"https://api.nuget.org/packages/automapper.1.1.0.118.nupkg","published":"2011-09-07T22:48:17.403+02:00","registration":"https://api.nuget.org/container1/automapper/index.json","@context":{"@vocab":"http://schema.nuget.org/schema#","xsd":"http://www.w3.org/2001/XMLSchema#","catalogEntry":{"@type":"@id"},"registration":{"@type":"@id"},"packageContent":{"@type":"@id"},"published":{"@type":"xsd:dateTime"}}} + + + {"@id":"https://api.nuget.org/container1/businessframework/index.json","@type":["catalog:CatalogRoot","PackageRegistration","catalog:Permalink"],"commitId":"2df1945a-8dd4-4b44-8330-527f4204f370","commitTimeStamp":"2016-02-26T09:22:09.4438705Z","count":1,"items":[{"@id":"https://api.nuget.org/container1/businessframework/index.json#page/0.2.0/0.2.0","@type":"catalog:CatalogPage","commitId":"2df1945a-8dd4-4b44-8330-527f4204f370","commitTimeStamp":"2016-02-26T09:22:09.4438705Z","count":1,"items":[{"@id":"https://api.nuget.org/container1/businessframework/0.2.0.json","@type":"Package","commitId":"2df1945a-8dd4-4b44-8330-527f4204f370","commitTimeStamp":"2016-02-26T09:22:09.4438705Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0-v2v3/data/2016.01.05.09.32.46/businessframework.0.2.0.json","@type":"PackageDetails","authors":"Michael Mac","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0-v2v3/data/2016.01.05.09.32.46/businessframework.0.2.0.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0-v2v3/data/2016.01.05.09.32.46/businessframework.0.2.0.json#dependencygroup/antlr","@type":"PackageDependency","id":"Antlr","range":"[3.1.1, )","registration":"https://api.nuget.org/container1/antlr/index.json"}]}],"description":"BusinessFrameowrk is a library which helps build business application. It provides rules, validation, textual lanugage - Formula, dynamic properties, etc.","iconUrl":"","id":"BusinessFramework","language":"en-US","licenseUrl":"","listed":true,"minClientVersion":"","packageContent":"https://api.nuget.org/packages/businessframework.0.2.0.nupkg","projectUrl":"","published":"2011-01-07T08:50:40.807+01:00","requireLicenseAcceptance":false,"summary":"","tags":[""],"title":"","version":"0.2.0"},"packageContent":"https://api.nuget.org/packages/businessframework.0.2.0.nupkg","registration":"https://api.nuget.org/container1/businessframework/index.json"}],"parent":"https://api.nuget.org/container1/businessframework/index.json","lower":"0.2.0","upper":"0.2.0"}],"@context":{"@vocab":"http://schema.nuget.org/schema#","catalog":"http://schema.nuget.org/catalog#","xsd":"http://www.w3.org/2001/XMLSchema#","items":{"@id":"catalog:item","@container":"@set"},"commitTimeStamp":{"@id":"catalog:commitTimeStamp","@type":"xsd:dateTime"},"commitId":{"@id":"catalog:commitId"},"count":{"@id":"catalog:count"},"parent":{"@id":"catalog:parent","@type":"@id"},"tags":{"@container":"@set","@id":"tag"},"packageTargetFrameworks":{"@container":"@set","@id":"packageTargetFramework"},"dependencyGroups":{"@container":"@set","@id":"dependencyGroup"},"dependencies":{"@container":"@set","@id":"dependency"},"packageContent":{"@type":"@id"},"published":{"@type":"xsd:dateTime"},"registration":{"@type":"@id"}}} + + + {"@id":"https://api.nuget.org/container1/businessframework/0.2.0.json","@type":["Package","http://schema.nuget.org/catalog#Permalink"],"catalogEntry":"https://api.nuget.org/v3/catalog0-v2v3/data/2016.01.05.09.32.46/businessframework.0.2.0.json","listed":true,"packageContent":"https://api.nuget.org/packages/businessframework.0.2.0.nupkg","published":"2011-01-07T08:50:40.807+01:00","registration":"https://api.nuget.org/container1/businessframework/index.json","@context":{"@vocab":"http://schema.nuget.org/schema#","xsd":"http://www.w3.org/2001/XMLSchema#","catalogEntry":{"@type":"@id"},"registration":{"@type":"@id"},"packageContent":{"@type":"@id"},"published":{"@type":"xsd:dateTime"}}} + + + { + "value": "2016-01-05T09:32:51.9339612" +} + + \ No newline at end of file diff --git a/tests/NgTests/Db2CatalogTests.cs b/tests/NgTests/Db2CatalogTests.cs new file mode 100644 index 000000000..acdb292df --- /dev/null +++ b/tests/NgTests/Db2CatalogTests.cs @@ -0,0 +1,1943 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Moq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NgTests.Data; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Versioning; +using Xunit; +using Xunit.Abstractions; + +namespace NgTests +{ + public class Db2CatalogTests : IDisposable + { + private const string PackageContentUrlFormat = "https://unittest.org/packages/{id-lower}/{version-lower}.nupkg"; + + private bool _isDisposed; + private DateTime _feedLastCreated; + private DateTime _feedLastEdited; + private DateTimeOffset _timestamp; + private bool _hasFirstRunOnceAsyncBeenCalledBefore; + private int _lastFeedEntriesCount; + private readonly List _packageOperations; + private readonly Random _random; + private readonly MemoryStorage _auditingStorage; + private readonly MemoryStorage _catalogStorage; + private TestableDb2CatalogJob _job; + private readonly Uri _baseUri; + private bool _skipCreatedPackagesProcessing; + private readonly MockServerHttpClientHandler _server; + private readonly PackageContentUriBuilder _packageContentUriBuilder; + private readonly ITestOutputHelper _testOutputHelper; + + public Db2CatalogTests(ITestOutputHelper testOutputHelper) + { + _testOutputHelper = testOutputHelper; + _server = new MockServerHttpClientHandler(); + _random = new Random(); + _packageOperations = new List(); + _baseUri = new Uri("http://unit.test"); + + _catalogStorage = new MemoryStorage(_baseUri); + _auditingStorage = new MemoryStorage(_baseUri); + + _packageContentUriBuilder = new PackageContentUriBuilder(PackageContentUrlFormat); + } + + public void Dispose() + { + if (!_isDisposed) + { + foreach (var operation in _packageOperations.OfType()) + { + operation.Package.Dispose(); + } + + GC.SuppressFinalize(this); + + _isDisposed = true; + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RunInternal_WithNoCatalogAndNoActivity_DoesNotCreateCatalog(bool skipCreatedPackagesProcessing) + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest(skipCreatedPackagesProcessing, top, galleryDatabaseMock); + + await RunInternalAndVerifyAsync(galleryDatabaseMock, top); + } + + [Fact] + public async Task RunInternal_WithNoCatalogAndCreatedPackageInFeed_CreatesCatalog() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: false, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); + } + + [Fact] + public async Task RunInternal_WithNoCatalogAndCreatedPackageInFeedAndWithCreatedPackagesSkipped_DoesNotCreateCatalog() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: true, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync(galleryDatabaseMock, top); + } + + [Fact] + public async Task RunInternal_WithCatalogAndNoActivity_DoesNotUpdateCatalog() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: false, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package = AddCreatedPackageToFeed(); + + // Create the catalog. + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); + + // Nothing new in the feed this time. + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); + } + + [Fact] + public async Task RunInternal_WithCatalogAndNoActivityAndWithCreatedPackagesSkipped_DoesNotUpdateCatalog() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: true, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package = CreatePackageCreationOrEdit(); + var editedPackage = AddEditedPackageToFeed(package); + + // Create the catalog. + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: editedPackage.FeedPackageDetails.LastEditedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: editedPackage.FeedPackageDetails.LastEditedDate); + + // Nothing new in the feed this time. + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: editedPackage.FeedPackageDetails.LastEditedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: editedPackage.FeedPackageDetails.LastEditedDate); + } + + [Fact] + // This test verifies current, flawed behavior. + // https://github.com/NuGet/NuGetGallery/issues/2841 + public async Task RunInternal_WithPackagesWithSameCreatedTimeInFeedAndWhenProcessedInDifferentCatalogBatches_SkipsSecondEntry() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: false, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package1 = AddCreatedPackageToFeed(); + var package2 = AddCreatedPackageToFeed(package1.FeedPackageDetails.CreatedDate); + + // Remove the "package2" argument if/when the bug is fixed. + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package1.FeedPackageDetails.CreatedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc, + skippedPackage: package2); + } + + [Fact] + // This test verifies current, flawed behavior. + // https://github.com/NuGet/NuGetGallery/issues/2841 + public async Task RunInternal_WithPackagesWithSameLastEditedTimeInFeedAndWhenProcessedInDifferentCatalogBatches_SkipsSecondEntry() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: false, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package1 = AddCreatedPackageToFeed(); + var package2 = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package2.FeedPackageDetails.CreatedDate, + expectedLastDeleted: package1.FeedPackageDetails.CreatedDate, + expectedLastEdited: Constants.DateTimeMinValueUtc); + + package1 = AddEditedPackageToFeed(package1); + package2 = AddEditedPackageToFeed(package2, package1.FeedPackageDetails.LastEditedDate); + + // Remove the "package2" argument if/when the bug is fixed. + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package2.FeedPackageDetails.CreatedDate, + expectedLastDeleted: package1.FeedPackageDetails.CreatedDate, + expectedLastEdited: package1.FeedPackageDetails.LastEditedDate, + skippedPackage: package2); + } + + [Fact] + public async Task RunInternal_WithCreatedPackagesInFeedAtDifferentTimes_UpdatesCatalog() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: false, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package1 = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package1.FeedPackageDetails.CreatedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); + + var package2 = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package2.FeedPackageDetails.CreatedDate, + expectedLastDeleted: package1.FeedPackageDetails.CreatedDate, + expectedLastEdited: Constants.DateTimeMinValueUtc); + } + + [Fact] + public async Task RunInternal_WithCreatedPackagesInFeedAtDifferentTimesAndWithCreatedPackagesSkipped_DoesNotUpdateCatalog() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: true, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync(galleryDatabaseMock, top); + + AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync(galleryDatabaseMock, top); + } + + [Fact] + public async Task RunInternal_WithCreatedPackageAndEditedPackageInFeedAtDifferentTimes_UpdatesCatalog() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: false, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); + + var editedPackage = AddEditedPackageToFeed(package); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: package.FeedPackageDetails.CreatedDate, + expectedLastEdited: editedPackage.FeedPackageDetails.LastEditedDate); + } + + [Fact] + public async Task RunInternal_WithCreatedPackageAndEditedPackageInFeedAtDifferentTimesAndWithCreatedPackagesSkipped_UpdatesCatalog() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: true, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); + + var editedPackage = AddEditedPackageToFeed(package); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: editedPackage.FeedPackageDetails.LastEditedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: editedPackage.FeedPackageDetails.LastEditedDate); + } + + [Fact] + public async Task RunInternal_WithCreatedPackageAndEditedPackageInFeedAtSameTime_UpdatesCatalog() + { + const int top = 1; + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: false, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); + + var editedPackage = AddEditedPackageToFeed(package); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: package.FeedPackageDetails.CreatedDate, + expectedLastEdited: editedPackage.FeedPackageDetails.LastEditedDate); + } + + [Fact] + public async Task RunInternal_WithCreatedPackageAndEditedPackageInFeedAtSameTimeAndWithCreatedPackagesSkipped_UpdatesCatalog() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: true, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); + + var editedPackage = AddEditedPackageToFeed(package); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: editedPackage.FeedPackageDetails.LastEditedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: editedPackage.FeedPackageDetails.LastEditedDate); + } + + [Fact] + public async Task RunInternal_WithEditedPackagesAndWithCreatedPackagesSkipped_UpdatesCatalog() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: true, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package = CreatePackageCreationOrEdit(); + var lastDeleted = Constants.DateTimeMinValueUtc; + + for (var i = 0; i < 3; ++i) + { + package = AddEditedPackageToFeed(package); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.LastEditedDate, + expectedLastDeleted: lastDeleted, + expectedLastEdited: package.FeedPackageDetails.LastEditedDate); + + if (lastDeleted == Constants.DateTimeMinValueUtc) + { + lastDeleted = package.FeedPackageDetails.LastEditedDate; + } + } + } + + [Fact] + public async Task RunInternal_WithCreatedPackageThenDeletedPackage_UpdatesCatalog() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: false, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); + + var deletedPackage = AddDeletedPackage(package); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: deletedPackage.DeletionTime.UtcDateTime, + expectedLastEdited: Constants.DateTimeMinValueUtc); + } + + [Fact] + public async Task RunInternal_WithMultipleDeletedPackagesWithDifferentPackageIdentities_ProcessesAllDeletions() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: false, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package1 = AddCreatedPackageToFeed(); + var package2 = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package2.FeedPackageDetails.CreatedDate, + expectedLastDeleted: package1.FeedPackageDetails.CreatedDate, + expectedLastEdited: Constants.DateTimeMinValueUtc); + + var deletedPackage1 = AddDeletedPackage(package1); + var deletedPackage2 = AddDeletedPackage(package2); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package2.FeedPackageDetails.CreatedDate, + expectedLastDeleted: deletedPackage2.DeletionTime.UtcDateTime, + expectedLastEdited: Constants.DateTimeMinValueUtc); + } + + [Fact] + public async Task RunInternal_WithMultipleDeletedPackagesWithSamePackageIdentity_PutsEachPackageInSeparateCommit() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: false, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); + + var deletionTime = DateTimeOffset.UtcNow; + + var deletedPackage1 = AddDeletedPackage(package, deletionTime.UtcDateTime, isSoftDelete: true); + var deletedPackage2 = AddDeletedPackage(package, deletionTime.UtcDateTime, isSoftDelete: false); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: deletionTime.UtcDateTime, + expectedLastEdited: Constants.DateTimeMinValueUtc); + } + + [Fact] + public async Task RunInternal_WithDeletedPackageOlderThan15MinutesAgo_SkipsDeletedPackage() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: false, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc); + + var deletedPackage = AddDeletedPackage(package, deletionTime: DateTime.UtcNow.AddMinutes(-16)); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package.FeedPackageDetails.CreatedDate, + expectedLastDeleted: Constants.DateTimeMinValueUtc, + expectedLastEdited: Constants.DateTimeMinValueUtc, + skippedPackage: deletedPackage); + } + + [Fact] + public async Task RunInternal_WithMultipleCreatedPackages_ProcessesAllCreations() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: false, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package1 = AddCreatedPackageToFeed(); + var package2 = AddCreatedPackageToFeed(); + + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package2.FeedPackageDetails.CreatedDate, + expectedLastDeleted: package1.FeedPackageDetails.CreatedDate, + expectedLastEdited: Constants.DateTimeMinValueUtc); + } + + [Fact] + public async Task RunInternal_WithMultipleEditedPackages_ProcessesAllEdits() + { + const int top = 1; + + var galleryDatabaseMock = new Mock(MockBehavior.Strict); + + InitializeTest( + skipCreatedPackagesProcessing: false, + top: top, + galleryDatabaseMock: galleryDatabaseMock); + + var package1 = AddCreatedPackageToFeed(); + var package2 = AddCreatedPackageToFeed(); + + // Create the catalog. + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package2.FeedPackageDetails.CreatedDate, + expectedLastDeleted: package1.FeedPackageDetails.CreatedDate, + expectedLastEdited: Constants.DateTimeMinValueUtc); + + var editedPackage1 = AddEditedPackageToFeed(package1); + var editedPackage2 = AddEditedPackageToFeed(package2); + + // Now test multiple edits. + await RunInternalAndVerifyAsync( + galleryDatabaseMock, + top, + expectedLastCreated: package2.FeedPackageDetails.CreatedDate, + expectedLastDeleted: package1.FeedPackageDetails.CreatedDate, + expectedLastEdited: editedPackage2.FeedPackageDetails.LastEditedDate); + } + + [Fact] + public async Task CreatesNewCatalogFromCreatedAndEditedAndDeletedPackages() + { + // Arrange + const int top = 20; + + var catalogStorage = new MemoryStorage(); + var auditingStorage = new MemoryStorage(); + auditingStorage.Content.TryAdd( + new Uri(auditingStorage.BaseAddress, "package/OtherPackage/1.0.0/2015-01-01T00:01:01-deleted.audit.v1.json"), + new StringStorageContent(TestCatalogEntries.DeleteAuditRecordForOtherPackage100)); + + var mockServer = new MockServerHttpClientHandler(); + RegisterPackageContentUri(mockServer, "ListedPackage", "1.0.0", "Packages\\ListedPackage.1.0.0.zip"); + RegisterPackageContentUri(mockServer, "ListedPackage", "1.0.1", "Packages\\ListedPackage.1.0.1.zip"); + RegisterPackageContentUri(mockServer, "UnlistedPackage", "1.0.0", "Packages\\UnlistedPackage.1.0.0.zip"); + RegisterPackageContentUri(mockServer, "TestPackage.SemVer2", "1.0.0-alpha.1", "Packages\\TestPackage.SemVer2.1.0.0-alpha.1.nupkg"); + + var cursor1 = new DateTime(1, 1, 1).ForceUtc(); + var cursor2 = new DateTime(2015, 1, 1).ForceUtc(); + + var galleryDatabaseMock = new Mock(); + + galleryDatabaseMock + .Setup(m => m.GetPackagesCreatedSince(cursor1, top)) + .ReturnsAsync(GetCreatedPackages); + + galleryDatabaseMock + .Setup(m => m.GetPackagesCreatedSince(cursor2, top)) + .ReturnsAsync(GetEmptyPackages); + + galleryDatabaseMock + .Setup(m => m.GetPackagesEditedSince(cursor1, top)) + .ReturnsAsync(GetEditedPackages); + + galleryDatabaseMock + .Setup(m => m.GetPackagesEditedSince(cursor2, top)) + .ReturnsAsync(GetEmptyPackages); + + // Act + var db2catalogTestJob = new TestableDb2CatalogJob( + mockServer, + catalogStorage, + auditingStorage, + skipCreatedPackagesProcessing: false, + startDate: null, + timeout: TimeSpan.FromMinutes(5), + top: top, + verbose: true, + galleryDatabaseMock: galleryDatabaseMock, + packageContentUriBuilder: _packageContentUriBuilder, + testOutputHelper: _testOutputHelper); + + await db2catalogTestJob.RunOnceAsync(CancellationToken.None); + + // Assert + Assert.Equal(7, catalogStorage.Content.Count); + + // Ensure catalog has index.json + var catalogIndex = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("index.json")); + Assert.NotNull(catalogIndex.Key); + Assert.Contains("\"nuget:lastCreated\":\"2015-01-01T00:00:00Z\"", catalogIndex.Value.GetContentString()); + Assert.Contains("\"nuget:lastDeleted\":\"2015-01-01T01:01:01.0748028Z\"", catalogIndex.Value.GetContentString()); + Assert.Contains("\"nuget:lastEdited\":\"2015-01-01T00:00:00Z\"", catalogIndex.Value.GetContentString()); + + // Ensure catalog has page0.json + var pageZero = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("page0.json")); + Assert.NotNull(pageZero.Key); + Assert.Contains("\"parent\":\"http://tempuri.org/index.json\",", pageZero.Value.GetContentString()); + + Assert.Contains("/listedpackage.1.0.0.json\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:id\":\"ListedPackage\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:version\":\"1.0.0\"", pageZero.Value.GetContentString()); + + Assert.Contains("/listedpackage.1.0.1.json\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:id\":\"ListedPackage\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:version\":\"1.0.1\"", pageZero.Value.GetContentString()); + + Assert.Contains("/unlistedpackage.1.0.0.json\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:id\":\"UnlistedPackage\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:version\":\"1.0.0\"", pageZero.Value.GetContentString()); + + Assert.Contains("/testpackage.semver2.1.0.0-alpha.1.json\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:id\":\"TestPackage.SemVer2\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:version\":\"1.0.0-alpha.1+githash\"", pageZero.Value.GetContentString()); + + // Check individual package entries + var package1 = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage.1.0.0.json")); + Assert.NotNull(package1.Key); + Assert.Contains("\"PackageDetails\",", package1.Value.GetContentString()); + Assert.Contains("\"id\": \"ListedPackage\",", package1.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.0\",", package1.Value.GetContentString()); + + var package2 = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage.1.0.1.json")); + Assert.NotNull(package2.Key); + Assert.Contains("\"PackageDetails\",", package2.Value.GetContentString()); + Assert.Contains("\"id\": \"ListedPackage\",", package2.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.1\",", package2.Value.GetContentString()); + + var package3 = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage.1.0.0.json")); + Assert.NotNull(package3.Key); + Assert.Contains("\"PackageDetails\",", package3.Value.GetContentString()); + Assert.Contains("\"id\": \"UnlistedPackage\",", package3.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.0\",", package3.Value.GetContentString()); + + var package4 = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2.1.0.0-alpha.1.json")); + Assert.NotNull(package4.Key); + Assert.Contains("\"PackageDetails\",", package4.Value.GetContentString()); + Assert.Contains("\"id\": \"TestPackage.SemVer2\",", package4.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.0-alpha.1+githash\",", package4.Value.GetContentString()); + + var package5 = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage.1.0.0.json")); + Assert.NotNull(package5.Key); + Assert.Contains("\"PackageDelete\",", package5.Value.GetContentString()); + Assert.Contains("\"id\": \"OtherPackage\",", package5.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.0\",", package5.Value.GetContentString()); + } + + [Fact] + public async Task AppendsDeleteToExistingCatalog() + { + // Arrange + const int top = 20; + var catalogStorage = Catalogs.CreateTestCatalogWithThreePackages(); + var auditingStorage = new MemoryStorage(); + + var firstAuditingRecord = new Uri(auditingStorage.BaseAddress, $"package/OtherPackage/1.0.0/{Guid.NewGuid()}-deleted.audit.v1.json"); + var secondAuditingRecord = new Uri(auditingStorage.BaseAddress, $"package/AnotherPackage/1.0.0/{Guid.NewGuid()}-deleted.audit.v1.json"); + + auditingStorage.Content.TryAdd(firstAuditingRecord, new StringStorageContent(TestCatalogEntries.DeleteAuditRecordForOtherPackage100)); + auditingStorage.Content.TryAdd(secondAuditingRecord, new StringStorageContent(TestCatalogEntries.DeleteAuditRecordForOtherPackage100.Replace("OtherPackage", "AnotherPackage"))); + auditingStorage.ListMock.TryAdd(secondAuditingRecord, new StorageListItem(secondAuditingRecord, new DateTime(2010, 1, 1))); + + var mockServer = new MockServerHttpClientHandler(); + RegisterPackageContentUri(mockServer, "ListedPackage", "1.0.0", "Packages\\ListedPackage.1.0.0.zip"); + RegisterPackageContentUri(mockServer, "ListedPackage", "1.0.1", "Packages\\ListedPackage.1.0.1.zip"); + RegisterPackageContentUri(mockServer, "UnlistedPackage", "1.0.0", "Packages\\UnlistedPackage.1.0.0.zip"); + + var cursor1 = new DateTime(1, 1, 1).ForceUtc(); + var cursor2 = new DateTime(2015, 1, 1).ForceUtc(); + + var galleryDatabaseMock = new Mock(); + + galleryDatabaseMock + .Setup(m => m.GetPackagesCreatedSince(cursor1, top)) + .ReturnsAsync(GetCreatedPackages); + + galleryDatabaseMock + .Setup(m => m.GetPackagesCreatedSince(cursor2, top)) + .ReturnsAsync(GetEmptyPackages); + + galleryDatabaseMock + .Setup(m => m.GetPackagesEditedSince(cursor1, top)) + .ReturnsAsync(GetEditedPackages); + + galleryDatabaseMock + .Setup(m => m.GetPackagesEditedSince(cursor2, top)) + .ReturnsAsync(GetEmptyPackages); + + // Act + var db2catalogTestJob = new TestableDb2CatalogJob( + mockServer, + catalogStorage, + auditingStorage, + skipCreatedPackagesProcessing: false, + startDate: null, + timeout: TimeSpan.FromMinutes(5), + top: top, + verbose: true, + galleryDatabaseMock: galleryDatabaseMock, + packageContentUriBuilder: _packageContentUriBuilder, + testOutputHelper: _testOutputHelper); + + await db2catalogTestJob.RunOnceAsync(CancellationToken.None); + + // Assert + Assert.Equal(6, catalogStorage.Content.Count); + + // Ensure catalog has index.json + var catalogIndex = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("index.json")); + Assert.NotNull(catalogIndex.Key); + Assert.Contains("\"nuget:lastCreated\":\"2015-01-01T00:00:00Z\"", catalogIndex.Value.GetContentString()); + Assert.Contains("\"nuget:lastDeleted\":\"2015-01-01T01:01:01", catalogIndex.Value.GetContentString()); + Assert.Contains("\"nuget:lastEdited\":\"2015-01-01T00:00:00", catalogIndex.Value.GetContentString()); + + // Ensure catalog has page0.json + var pageZero = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("page0.json")); + Assert.NotNull(pageZero.Key); + Assert.Contains("\"parent\":\"http://tempuri.org/index.json\",", pageZero.Value.GetContentString()); + + Assert.Contains("/listedpackage.1.0.0.json\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:id\":\"ListedPackage\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:version\":\"1.0.0\"", pageZero.Value.GetContentString()); + + Assert.Contains("/listedpackage.1.0.1.json\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:id\":\"ListedPackage\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:version\":\"1.0.1\"", pageZero.Value.GetContentString()); + + Assert.Contains("/unlistedpackage.1.0.0.json\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:id\":\"UnlistedPackage\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:version\":\"1.0.0\"", pageZero.Value.GetContentString()); + + Assert.Contains("/otherpackage.1.0.0.json\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:id\":\"OtherPackage\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:version\":\"1.0.0\"", pageZero.Value.GetContentString()); + + // Check individual package entries + var package1 = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage.1.0.0.json")); + Assert.NotNull(package1.Key); + Assert.Contains("\"PackageDetails\",", package1.Value.GetContentString()); + Assert.Contains("\"id\": \"ListedPackage\",", package1.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.0\",", package1.Value.GetContentString()); + + var package2 = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage.1.0.1.json")); + Assert.NotNull(package2.Key); + Assert.Contains("\"PackageDetails\",", package2.Value.GetContentString()); + Assert.Contains("\"id\": \"ListedPackage\",", package2.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.1\",", package2.Value.GetContentString()); + + var package3 = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage.1.0.0.json")); + Assert.NotNull(package3.Key); + Assert.Contains("\"PackageDetails\",", package3.Value.GetContentString()); + Assert.Contains("\"id\": \"UnlistedPackage\",", package3.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.0\",", package3.Value.GetContentString()); + + // Ensure catalog has the delete of "OtherPackage" + var package4 = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage.1.0.0.json")); + Assert.NotNull(package4.Key); + Assert.Contains("\"PackageDelete\",", package4.Value.GetContentString()); + Assert.Contains("\"id\": \"OtherPackage\",", package4.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.0\",", package4.Value.GetContentString()); + } + + [Fact] + public async Task AppendsDeleteAndReinsertToExistingCatalog() + { + // Arrange + var top = 20; + var catalogStorage = Catalogs.CreateTestCatalogWithThreePackages(); + var auditingStorage = new MemoryStorage(); + auditingStorage.Content.TryAdd( + new Uri(auditingStorage.BaseAddress, "package/OtherPackage/1.0.0/2015-01-01T00:01:01-deleted.audit.v1.json"), + new StringStorageContent(TestCatalogEntries.DeleteAuditRecordForOtherPackage100)); + + var mockServer = new MockServerHttpClientHandler(); + RegisterPackageContentUri(mockServer, "ListedPackage", "1.0.0", "Packages\\ListedPackage.1.0.0.zip"); + RegisterPackageContentUri(mockServer, "ListedPackage", "1.0.1", "Packages\\ListedPackage.1.0.1.zip"); + RegisterPackageContentUri(mockServer, "UnlistedPackage", "1.0.0", "Packages\\UnlistedPackage.1.0.0.zip"); + RegisterPackageContentUri(mockServer, "OtherPackage", "1.0.0", "Packages\\OtherPackage.1.0.0.zip"); + + var cursor1 = new DateTime(1, 1, 1).ForceUtc(); + var cursor2 = new DateTime(2015, 1, 1).ForceUtc(); + var cursor3 = new DateTime(2015, 1, 1, 1, 1, 3).ForceUtc(); + + var galleryDatabaseMock = new Mock(); + + galleryDatabaseMock + .Setup(m => m.GetPackagesCreatedSince(cursor1, top)) + .ReturnsAsync(GetCreatedPackages); + + galleryDatabaseMock + .Setup(m => m.GetPackagesCreatedSince(cursor2, top)) + .ReturnsAsync(GetCreatedPackagesSecondRequest); + + galleryDatabaseMock + .Setup(m => m.GetPackagesCreatedSince(cursor3, top)) + .ReturnsAsync(GetEmptyPackages); + + galleryDatabaseMock + .Setup(m => m.GetPackagesEditedSince(cursor1, top)) + .ReturnsAsync(GetEditedPackages); + + galleryDatabaseMock + .Setup(m => m.GetPackagesEditedSince(cursor2, top)) + .ReturnsAsync(GetEmptyPackages); + + // Act + var db2catalogTestJob = new TestableDb2CatalogJob( + mockServer, + catalogStorage, + auditingStorage, + skipCreatedPackagesProcessing: false, + startDate: null, + timeout: TimeSpan.FromMinutes(5), + top: top, + verbose: true, + galleryDatabaseMock: galleryDatabaseMock, + packageContentUriBuilder: _packageContentUriBuilder, + testOutputHelper: _testOutputHelper); + + await db2catalogTestJob.RunOnceAsync(CancellationToken.None); + + // Assert + Assert.Equal(7, catalogStorage.Content.Count); + + // Ensure catalog has index.json + var catalogIndex = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("index.json")); + Assert.NotNull(catalogIndex.Key); + Assert.Contains("\"nuget:lastCreated\":\"2015-01-01T01:01:03Z\"", catalogIndex.Value.GetContentString()); + Assert.Contains("\"nuget:lastDeleted\":\"2015-01-01T01:01:01", catalogIndex.Value.GetContentString()); + Assert.Contains("\"nuget:lastEdited\":\"2015-01-01T00:00:00", catalogIndex.Value.GetContentString()); + + // Ensure catalog has page0.json + var pageZero = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("page0.json")); + Assert.NotNull(pageZero.Key); + Assert.Contains("\"parent\":\"http://tempuri.org/index.json\",", pageZero.Value.GetContentString()); + + Assert.Contains("/listedpackage.1.0.0.json\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:id\":\"ListedPackage\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:version\":\"1.0.0\"", pageZero.Value.GetContentString()); + + Assert.Contains("/listedpackage.1.0.1.json\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:id\":\"ListedPackage\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:version\":\"1.0.1\"", pageZero.Value.GetContentString()); + + Assert.Contains("/unlistedpackage.1.0.0.json\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:id\":\"UnlistedPackage\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:version\":\"1.0.0\"", pageZero.Value.GetContentString()); + + Assert.Contains("/otherpackage.1.0.0.json\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:id\":\"OtherPackage\",", pageZero.Value.GetContentString()); + Assert.Contains("\"nuget:version\":\"1.0.0\"", pageZero.Value.GetContentString()); + + // Check individual package entries + var package1 = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage.1.0.0.json")); + Assert.NotNull(package1.Key); + Assert.Contains("\"PackageDetails\",", package1.Value.GetContentString()); + Assert.Contains("\"id\": \"ListedPackage\",", package1.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.0\",", package1.Value.GetContentString()); + + var package2 = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage.1.0.1.json")); + Assert.NotNull(package2.Key); + Assert.Contains("\"PackageDetails\",", package2.Value.GetContentString()); + Assert.Contains("\"id\": \"ListedPackage\",", package2.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.1\",", package2.Value.GetContentString()); + + var package3 = catalogStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage.1.0.0.json")); + Assert.NotNull(package3.Key); + Assert.Contains("\"PackageDetails\",", package3.Value.GetContentString()); + Assert.Contains("\"id\": \"UnlistedPackage\",", package3.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.0\",", package3.Value.GetContentString()); + + // Ensure catalog has the delete of "OtherPackage" + var package4 = catalogStorage.Content.FirstOrDefault(pair => + pair.Key.PathAndQuery.EndsWith("/otherpackage.1.0.0.json") + && pair.Value.GetContentString().Contains("\"PackageDelete\"")); + Assert.NotNull(package4.Key); + Assert.Contains("\"PackageDelete\",", package4.Value.GetContentString()); + Assert.Contains("\"id\": \"OtherPackage\",", package4.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.0\",", package4.Value.GetContentString()); + + // Ensure catalog has the insert of "OtherPackage" + var package5 = catalogStorage.Content.FirstOrDefault(pair => + pair.Key.PathAndQuery.EndsWith("/otherpackage.1.0.0.json") + && pair.Value.GetContentString().Contains("\"PackageDetails\"")); + Assert.NotNull(package5.Key); + Assert.Contains("\"PackageDetails\",", package5.Value.GetContentString()); + Assert.Contains("\"id\": \"OtherPackage\",", package5.Value.GetContentString()); + Assert.Contains("\"version\": \"1.0.0\",", package5.Value.GetContentString()); + } + + [Fact] + public async Task RunInternal_CallsCatalogStorageLoadStringExactlyOnce() + { + const int top = 20; + var mockServer = new MockServerHttpClientHandler(); + var auditingStorage = Mock.Of(); + var catalogStorage = new Mock(MockBehavior.Strict); + var datetime = DateTime.MinValue.ToString("O") + "Z"; + var json = $"{{\"nuget:lastCreated\":\"{datetime}\"," + + $"\"nuget:lastDeleted\":\"{datetime}\"," + + $"\"nuget:lastEdited\":\"{datetime}\"}}"; + + catalogStorage.Setup(x => x.ResolveUri(It.IsNotNull())) + .Returns(_baseUri); + catalogStorage.Setup(x => x.LoadStringAsync(It.IsNotNull(), It.IsAny())) + .ReturnsAsync(json); + + var cursor1 = new DateTime(1, 1, 1).ForceUtc(); + + var galleryDatabaseMock = new Mock(); + + galleryDatabaseMock + .Setup(m => m.GetPackagesCreatedSince(cursor1, top)) + .ReturnsAsync(GetEmptyPackages); + + galleryDatabaseMock + .Setup(m => m.GetPackagesEditedSince(cursor1, top)) + .ReturnsAsync(GetEmptyPackages); + + var db2catalogTestJob = new TestableDb2CatalogJob( + mockServer, + catalogStorage.Object, + auditingStorage, + skipCreatedPackagesProcessing: false, + startDate: null, + timeout: TimeSpan.FromMinutes(5), + top: top, + verbose: true, + galleryDatabaseMock: galleryDatabaseMock, + packageContentUriBuilder: _packageContentUriBuilder, + testOutputHelper: _testOutputHelper); + + await db2catalogTestJob.RunOnceAsync(CancellationToken.None); + + catalogStorage.Verify(x => x.ResolveUri(It.IsNotNull()), Times.AtLeastOnce()); + catalogStorage.Verify(x => x.LoadStringAsync(It.IsNotNull(), It.IsAny()), Times.Once()); + } + + private SortedList> GetCreatedPackages() + { + var packages = new List + { + new FeedPackageDetails( + contentUri: _packageContentUriBuilder.Build("ListedPackage", "1.0.0"), + createdDate: new DateTime(2015, 1, 1).ForceUtc(), + lastEditedDate: DateTime.MinValue, + publishedDate: new DateTime(2015, 1, 1).ForceUtc(), + packageId: "ListedPackage", + packageNormalizedVersion: "1.0.0", + packageFullVersion: "1.0.0.0"), + new FeedPackageDetails( + contentUri: _packageContentUriBuilder.Build("UnlistedPackage", "1.0.0"), + createdDate: new DateTime(2015, 1, 1).ForceUtc(), + lastEditedDate: DateTime.MinValue, + publishedDate: Constants.UnpublishedDate, + packageId: "UnlistedPackage", + packageNormalizedVersion: "1.0.0", + packageFullVersion: "1.0.0+metadata"), + + // The real SemVer2 version is embedded in the nupkg. + // The below FeedPackageDetails entity expects normalized versions. + new FeedPackageDetails( + contentUri: _packageContentUriBuilder.Build("TestPackage.SemVer2", "1.0.0-alpha.1"), + createdDate: new DateTime(2015, 1, 1).ForceUtc(), + lastEditedDate: DateTime.MinValue, + publishedDate: new DateTime(2015, 1, 1).ForceUtc(), + packageId: "TestPackage.SemVer2", + packageNormalizedVersion: "1.0.0-alpha.1", + packageFullVersion: "1.0.0-alpha.1") + }; + + return GalleryDatabaseQueryService.OrderPackagesByKeyDate(packages, p => p.CreatedDate); + } + + private SortedList> GetEditedPackages() + { + var packages = new List + { + new FeedPackageDetails( + contentUri: _packageContentUriBuilder.Build("ListedPackage", "1.0.1"), + createdDate: new DateTime(2014, 1, 1).ForceUtc(), + lastEditedDate: new DateTime(2015, 1, 1).ForceUtc(), + publishedDate: new DateTime(2014, 1, 1).ForceUtc(), + packageId: "ListedPackage", + packageNormalizedVersion: "1.0.1", + packageFullVersion: "1.0.1") + }; + + return GalleryDatabaseQueryService.OrderPackagesByKeyDate(packages, p => p.LastEditedDate); + } + + private SortedList> GetEmptyPackages() + { + return GalleryDatabaseQueryService.OrderPackagesByKeyDate(new List(), p => p.CreatedDate); + } + + private SortedList> GetCreatedPackagesSecondRequest() + { + var packages = new List + { + new FeedPackageDetails( + contentUri: _packageContentUriBuilder.Build("OtherPackage", "1.0.0"), + createdDate: new DateTime(2015, 1, 1, 1, 1, 3).ForceUtc(), + lastEditedDate: new DateTime(2015, 1, 1, 1, 1, 3).ForceUtc(), + publishedDate: new DateTime(2015, 1, 1, 1, 1, 3).ForceUtc(), + packageId: "OtherPackage", + packageNormalizedVersion: "1.0.0", + packageFullVersion: "1.0.0") + }; + + return GalleryDatabaseQueryService.OrderPackagesByKeyDate(packages, p => p.CreatedDate); + } + + private void InitializeTest( + bool skipCreatedPackagesProcessing, + int top, + Mock galleryDatabaseMock) + { + _skipCreatedPackagesProcessing = skipCreatedPackagesProcessing; + + _job = new TestableDb2CatalogJob( + _server, + _catalogStorage, + _auditingStorage, + skipCreatedPackagesProcessing, + startDate: null, + timeout: TimeSpan.FromMinutes(5), + top: top, + verbose: true, + galleryDatabaseMock: galleryDatabaseMock, + packageContentUriBuilder: _packageContentUriBuilder, + testOutputHelper: _testOutputHelper); + } + + private PackageCreationOrEdit CreatePackageCreationOrEdit(DateTime? createdDate = null) + { + var package = TestPackage.Create(_random); + var isListed = Convert.ToBoolean(_random.Next(minValue: 0, maxValue: 2)); + var created = createdDate ?? DateTimeOffset.UtcNow; + + // Avoid hitting this bug accidentally: https://github.com/NuGet/NuGetGallery/issues/2841 + if (!createdDate.HasValue && created == _packageOperations.OfType().LastOrDefault()?.FeedPackageDetails.CreatedDate) + { + created = created.AddMilliseconds(1); + } + + var normalizedVersion = package.Version.ToNormalizedString(); + var fullVersion = package.Version.ToFullString(); + var feedPackageDetails = new FeedPackageDetails( + contentUri: _packageContentUriBuilder.Build(package.Id, normalizedVersion), + createdDate: created.UtcDateTime, + lastEditedDate: DateTime.MinValue, + publishedDate: isListed ? created.UtcDateTime : Constants.UnpublishedDate, + packageId: package.Id, + packageNormalizedVersion: normalizedVersion, + packageFullVersion: fullVersion, + licenseNames: null, + licenseReportUrl: null, + deprecationInfo: null, + requiresLicenseAcceptance: false) ; + + return new PackageCreationOrEdit(package, feedPackageDetails); + } + + private PackageCreationOrEdit AddCreatedPackageToFeed(DateTime? createdDate = null) + { + var operation = CreatePackageCreationOrEdit(createdDate); + + _packageOperations.Add(operation); + + return operation; + } + + private PackageCreationOrEdit AddEditedPackageToFeed(PackageCreationOrEdit entry, DateTime? lastEditedDate = null) + { + var editedPackage = AddPackageEntry(entry.Package); + var edited = lastEditedDate ?? DateTime.UtcNow; + + // Avoid hitting this bug accidentally: https://github.com/NuGet/NuGetGallery/issues/2841 + if (!lastEditedDate.HasValue && edited == _packageOperations.OfType().LastOrDefault(e => e.FeedPackageDetails.LastEditedDate != DateTime.MinValue)?.FeedPackageDetails.LastEditedDate) + { + edited = edited.AddMilliseconds(1); + } + + var feedPackageDetails = new FeedPackageDetails( + contentUri: entry.FeedPackageDetails.ContentUri, + createdDate: entry.FeedPackageDetails.CreatedDate, + lastEditedDate: edited, + publishedDate: entry.FeedPackageDetails.PublishedDate, + packageId: entry.FeedPackageDetails.PackageId, + packageNormalizedVersion: entry.FeedPackageDetails.PackageNormalizedVersion, + packageFullVersion: entry.FeedPackageDetails.PackageFullVersion, + licenseNames: entry.FeedPackageDetails.LicenseNames, + licenseReportUrl: entry.FeedPackageDetails.LicenseReportUrl, + deprecationInfo: entry.FeedPackageDetails.DeprecationInfo, + requiresLicenseAcceptance: entry.FeedPackageDetails.RequiresLicenseAcceptance); + + var operation = new PackageCreationOrEdit(editedPackage, feedPackageDetails); + + _packageOperations.Add(operation); + + return operation; + } + + private PackageDeletion AddDeletedPackage( + PackageCreationOrEdit operation, + DateTime? deletionTime = null, + bool isSoftDelete = false) + { + // By default, avoid the deletion time being equal to the catalog's last deleted cursor. + deletionTime = deletionTime ?? GetUniqueDateTime(); + + var auditRecord = new JObject( + new JProperty("record", + new JObject( + new JProperty("id", operation.Package.Id), + new JProperty("version", operation.Package.Version.ToNormalizedString()))), + new JProperty("actor", + new JObject( + new JProperty("timestampUtc", deletionTime.Value.ToString("O"))))); + + var packageId = operation.Package.Id.ToLowerInvariant(); + var packageVersion = operation.Package.Version.ToNormalizedString().ToLowerInvariant(); + var fileNamePostfix = isSoftDelete ? "softdelete.audit.v1.json" : "Deleted.audit.v1.json"; + var uri = new Uri($"https://nuget.test/auditing/{packageId}/{packageVersion}-{fileNamePostfix}"); + + _auditingStorage.Content.TryAdd(uri, new JTokenStorageContent(auditRecord)); + + var deletion = new PackageDeletion(uri, auditRecord, deletionTime.Value); + + _packageOperations.Add(deletion); + + return deletion; + } + + private async Task RunInternalAndVerifyAsync( + Mock galleryDatabaseHelperMock, + int top, + DateTime? expectedLastCreated = null, + DateTime? expectedLastDeleted = null, + DateTime? expectedLastEdited = null, + PackageOperation skippedPackage = null) + { + if (_hasFirstRunOnceAsyncBeenCalledBefore) + { + // Package details URL's contain a timestamp with second granularity. + // Format: /data/yyyy.MM.dd.HH.mm.ss/..json + // Example: https://nuget.test/data/2018.07.20.20.15.30/bjxobigmgsxsossw.3.8.9.json + // To ensure that successive catalog batches have unique timestamps, wait one full second. + await Task.Delay(TimeSpan.FromSeconds(1)); + } + + _hasFirstRunOnceAsyncBeenCalledBefore = true; + + _server.Actions.Clear(); + + PublishCreatedPackages(galleryDatabaseHelperMock, top); + PublishEditedPackages(galleryDatabaseHelperMock, top); + + await _job.RunOnceAsync(CancellationToken.None); + + VerifyCatalog(expectedLastCreated, expectedLastDeleted, expectedLastEdited, skippedPackage); + } + + private void RegisterPackageContentUri(TestPackage package) + { + _server.SetAction( + _packageContentUriBuilder.Build(package.Id, package.Version.ToNormalizedString()).AbsolutePath, + request => GetStreamContentActionAsync(package.Stream)); + } + + private void RegisterPackageContentUri(MockServerHttpClientHandler server, string packageId, string packageVersion, string filePath) + { + server.SetAction( + _packageContentUriBuilder.Build(packageId, packageVersion).AbsolutePath, + request => GetStreamContentActionAsync(filePath)); + } + + private void VerifyCatalog( + DateTime? expectedLastCreated, + DateTime? expectedLastDeleted, + DateTime? expectedLastEdited, + PackageOperation skippedOperation) + { + List verifiablePackageOperations; + + if (_skipCreatedPackagesProcessing) + { + verifiablePackageOperations = _packageOperations + .Where(entry => + entry != skippedOperation + && (!(entry is PackageCreationOrEdit) || ((PackageCreationOrEdit)entry).FeedPackageDetails.LastEditedDate != DateTime.MinValue)) + .ToList(); + } + else + { + verifiablePackageOperations = _packageOperations + .Where(entry => entry != skippedOperation) + .ToList(); + } + + if (verifiablePackageOperations.Count == 0) + { + Assert.Empty(_catalogStorage.Content); + + return; + } + + bool isEmptyCatalogBatch = _lastFeedEntriesCount == verifiablePackageOperations.Count; + + if (!isEmptyCatalogBatch) + { + _lastFeedEntriesCount = verifiablePackageOperations.Count; + } + + var expectedCatalogEntryCount = verifiablePackageOperations.Count + + 1 // index.json + + 1; // page0.json + Assert.Equal(expectedCatalogEntryCount, _catalogStorage.Content.Count); + + Assert.True(expectedLastCreated.HasValue); + Assert.True(expectedLastDeleted.HasValue); + Assert.True(expectedLastEdited.HasValue); + + var indexUri = new Uri(_baseUri, "index.json"); + var pageUri = new Uri(_baseUri, "page0.json"); + + string commitId; + string commitTimeStamp; + + VerifyCatalogIndex( + verifiablePackageOperations, + indexUri, + pageUri, + expectedLastCreated.Value, + expectedLastDeleted.Value, + expectedLastEdited.Value, + out commitId, + out commitTimeStamp); + + Assert.True( + DateTime.TryParseExact( + commitTimeStamp, + CatalogConstants.CommitTimeStampFormat, + DateTimeFormatInfo.CurrentInfo, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var commitTimeStampDateTime)); + + VerifyCatalogPage(verifiablePackageOperations, indexUri, pageUri, commitId, commitTimeStamp); + VerifyCatalogPackageItems(verifiablePackageOperations); + + Assert.True(verifiablePackageOperations.All(packageOperation => !string.IsNullOrEmpty(packageOperation.CommitId))); + } + + private void VerifyCatalogIndex( + List packageOperations, + Uri indexUri, + Uri pageUri, + DateTime expectedLastCreated, + DateTime expectedLastDeleted, + DateTime expectedLastEdited, + out string commitId, + out string commitTimeStamp) + { + Assert.True(_catalogStorage.Content.TryGetValue(indexUri, out var storage)); + Assert.IsType(storage); + + var index = ((JTokenStorageContent)storage).Content as JObject; + Assert.NotNull(index); + + index = ReadJsonWithoutDateTimeHandling(index); + + var properties = index.Properties(); + + Assert.Equal(10, properties.Count()); + Assert.Equal(indexUri.AbsoluteUri, index[CatalogConstants.IdKeyword].Value()); + Assert.Equal( + new JArray(CatalogConstants.CatalogRoot, CatalogConstants.AppendOnlyCatalog, CatalogConstants.Permalink).ToString(), + index[CatalogConstants.TypeKeyword].ToString()); + + commitId = index[CatalogConstants.CommitId].Value(); + Assert.True(Guid.TryParse(commitId, out var guid)); + + commitTimeStamp = index[CatalogConstants.CommitTimeStamp].Value(); + + Assert.Equal(1, index[CatalogConstants.Count].Value()); + + Assert.Equal( + expectedLastCreated.ToString("O"), + DateTime.Parse(index[CatalogConstants.NuGetLastCreated].Value(), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind).ToString("O")); + Assert.Equal( + expectedLastDeleted.ToString("O"), + DateTime.Parse(index[CatalogConstants.NuGetLastDeleted].Value(), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind).ToString("O")); + Assert.Equal( + expectedLastEdited.ToString("O"), + DateTime.Parse(index[CatalogConstants.NuGetLastEdited].Value(), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind).ToString("O")); + + var expectedItems = new JArray( + new JObject( + new JProperty(CatalogConstants.IdKeyword, pageUri.AbsoluteUri), + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.CatalogPage), + new JProperty(CatalogConstants.CommitId, commitId), + new JProperty(CatalogConstants.CommitTimeStamp, commitTimeStamp), + new JProperty(CatalogConstants.Count, packageOperations.Count))); + + Assert.Equal(expectedItems.ToString(), index[CatalogConstants.Items].ToString()); + + VerifyContext(index); + } + + private void VerifyCatalogPage( + List packageOperations, + Uri indexUri, + Uri pageUri, + string commitId, + string commitTimeStamp) + { + Assert.True(_catalogStorage.Content.TryGetValue(pageUri, out var storage)); + Assert.IsType(storage); + + var page = ((JTokenStorageContent)storage).Content as JObject; + Assert.NotNull(page); + + page = ReadJsonWithoutDateTimeHandling(page); + + var properties = page.Properties(); + + Assert.Equal(8, properties.Count()); + Assert.Equal(pageUri.AbsoluteUri, page[CatalogConstants.IdKeyword].Value()); + Assert.Equal(CatalogConstants.CatalogPage, page[CatalogConstants.TypeKeyword].Value()); + + Assert.Equal(commitId, page[CatalogConstants.CommitId].Value()); + Assert.Equal(commitTimeStamp, page[CatalogConstants.CommitTimeStamp].Value()); + + var actualPackageDetailsCount = page[CatalogConstants.Count].Value(); + + Assert.Equal(packageOperations.Count, actualPackageDetailsCount); + + Assert.Equal(indexUri.AbsoluteUri, page[CatalogConstants.Parent].Value()); + + // This is a bit tricky. + // A catalog page lists package detail items in chronologically descending order, + // which is the reverse order of our internal list. + // https://github.com/NuGet/NuGetGallery/issues/4757 + // Also, within a catalog batch the order of individual items is nondeterministic. + // We'll match up expected items with actual items by picking in the next commit batch + // the first item we haven't seen already with the same package identity as the expected item. + var items = page[CatalogConstants.Items].Reverse().ToArray(); + var itemsByPackageIdentity = items + .GroupBy(item => new PackageIdentity( + item[CatalogConstants.NuGetId].Value(), + new NuGetVersion(item[CatalogConstants.NuGetVersion].Value()))) + .ToArray(); + var actualItemsCount = items.Count(); + Assert.Equal(packageOperations.Count, actualItemsCount); + + var unseenItems = new HashSet(items); + + for (var i = 0; i < actualItemsCount; ++i) + { + var expectedItem = packageOperations[i]; + var actualItem = itemsByPackageIdentity.Single(group => group.Key.Equals(expectedItem.PackageIdentity)) + .First(item => unseenItems.Contains(item)); + + unseenItems.Remove(actualItem); + + var actualCommitId = actualItem[CatalogConstants.CommitId].Value(); + var actualCommitTimeStamp = actualItem[CatalogConstants.CommitTimeStamp].Value(); + + if (i == actualItemsCount - 1) + { + Assert.Equal(commitId, actualCommitId); + Assert.Equal(commitTimeStamp, actualCommitTimeStamp); + } + + // If there are later catalog updates, we'll verify these haven't changed then. + expectedItem.CommitId = expectedItem.CommitId ?? actualCommitId; + expectedItem.CommitTimeStamp = expectedItem.CommitTimeStamp ?? actualCommitTimeStamp; + + var expectedTimestamp = DateTime.ParseExact( + expectedItem.CommitTimeStamp, + CatalogConstants.CommitTimeStampFormat, + DateTimeFormatInfo.CurrentInfo, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal); + + expectedItem.CommitTimeStampDateTime = expectedTimestamp; + + var expectedUri = GetPackageDetailsUri(expectedTimestamp, expectedItem); + + Assert.Equal(expectedUri.AbsoluteUri, actualItem[CatalogConstants.IdKeyword].Value()); + + var expectedType = expectedItem is PackageCreationOrEdit ? CatalogConstants.NuGetPackageDetails : CatalogConstants.NuGetPackageDelete; + + Assert.Equal(expectedType, actualItem[CatalogConstants.TypeKeyword].Value()); + Assert.Equal(expectedItem.CommitId, actualCommitId); + Assert.Equal(expectedItem.CommitTimeStamp, actualCommitTimeStamp); + Assert.Equal(expectedItem.PackageIdentity.Id, actualItem[CatalogConstants.NuGetId].Value()); + Assert.Equal( + expectedItem.PackageIdentity.Version.ToNormalizedString(), + actualItem[CatalogConstants.NuGetVersion].Value()); + } + + VerifyContext(page); + } + + private void VerifyCatalogPackageItems(List packageOperations) + { + PackageOperation previousPackageOperation = null; + + foreach (var packageOperation in packageOperations) + { + if (previousPackageOperation != null) + { + Assert.True(packageOperation.CommitTimeStampDateTime >= previousPackageOperation.CommitTimeStampDateTime); + } + + Uri packageDetailsUri = GetPackageDetailsUri(packageOperation.CommitTimeStampDateTime, packageOperation); + + Assert.True(_catalogStorage.Content.TryGetValue(packageDetailsUri, out var storage)); + Assert.IsAssignableFrom(storage); + + var packageDetails = JObject.Parse(((StringStorageContent)storage).Content); + Assert.NotNull(packageDetails); + + packageDetails = ReadJsonWithoutDateTimeHandling(packageDetails); + + if (packageOperation is PackageCreationOrEdit) + { + VerifyCatalogPackageDetails((PackageCreationOrEdit)packageOperation, packageDetailsUri, packageDetails); + } + else + { + VerifyCatalogPackageDelete((PackageDeletion)packageOperation, packageDetailsUri, packageDetails); + } + + previousPackageOperation = packageOperation; + } + } + + private void VerifyCatalogPackageDetails( + PackageCreationOrEdit packageOperation, + Uri packageDetailsUri, + JObject packageDetails) + { + var properties = packageDetails.Properties(); + + Assert.Equal(19, properties.Count()); + Assert.Equal(packageDetailsUri.AbsoluteUri, packageDetails[CatalogConstants.IdKeyword].Value()); + Assert.Equal( + new JArray(CatalogConstants.PackageDetails, CatalogConstants.CatalogPermalink), + packageDetails[CatalogConstants.TypeKeyword]); + Assert.Equal(packageOperation.Package.Author, packageDetails[CatalogConstants.Authors].Value()); + Assert.Equal(packageOperation.CommitId, packageDetails[CatalogConstants.CatalogCommitId].Value()); + Assert.Equal(packageOperation.CommitTimeStamp, packageDetails[CatalogConstants.CatalogCommitTimeStamp].Value()); + Assert.Equal( + packageOperation.FeedPackageDetails.CreatedDate.ToString("O"), + DateTime.Parse(packageDetails[CatalogConstants.Created].Value(), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind).ToString("O")); + Assert.Equal(packageOperation.Package.Description, packageDetails[CatalogConstants.Description].Value()); + Assert.Equal(packageOperation.FeedPackageDetails.PackageId, packageDetails[CatalogConstants.Id].Value()); + Assert.False(packageDetails[CatalogConstants.IsPrerelease].Value()); + Assert.Equal( + packageOperation.FeedPackageDetails.LastEditedDate.ToString("O"), + DateTime.Parse(packageDetails[CatalogConstants.LastEdited].Value(), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind).ToString("O")); + Assert.Equal(packageOperation.FeedPackageDetails.PublishedDate != Constants.UnpublishedDate, packageDetails[CatalogConstants.Listed].Value()); + Assert.Equal(GetPackageHash(packageOperation.Package), packageDetails[CatalogConstants.PackageHash].Value()); + Assert.Equal(Constants.Sha512, packageDetails[CatalogConstants.PackageHashAlgorithm].Value()); + Assert.Equal(packageOperation.Package.Stream.Length, packageDetails[CatalogConstants.PackageSize].Value()); + Assert.Equal( + packageOperation.FeedPackageDetails.PublishedDate.ToString("O"), + DateTime.Parse(packageDetails[CatalogConstants.Published].Value(), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind).ToString("O")); + Assert.Equal(packageOperation.Package.Version.ToFullString(), packageDetails[CatalogConstants.VerbatimVersion].Value()); + Assert.Equal(packageOperation.Package.Version.ToNormalizedString(), packageDetails[CatalogConstants.Version].Value()); + + var expectedPackageEntries = GetPackageEntries(packageOperation.Package) + .OrderBy(entry => entry.FullName) + .Select(entry => + new JObject( + new JProperty(CatalogConstants.IdKeyword, new Uri(packageDetailsUri, $"#{entry.FullName}").AbsoluteUri), + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.PackageEntry), + new JProperty(CatalogConstants.CompressedLength, entry.CompressedLength), + new JProperty(CatalogConstants.FullName, entry.FullName), + new JProperty(CatalogConstants.Length, entry.Length), + new JProperty(CatalogConstants.Name, entry.Name))) + .ToArray(); + + var actualPackageEntries = packageDetails[CatalogConstants.PackageEntries] + .Children() + .OrderBy(token => token[CatalogConstants.FullName].Value()) + .ToArray(); + + Assert.Equal(expectedPackageEntries.Length, actualPackageEntries.Length); + + for (var i = 0; i < expectedPackageEntries.Length; ++i) + { + Assert.Equal(expectedPackageEntries[i].ToString(), actualPackageEntries[i].ToString()); + } + + var expectedContext = new JObject( + new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetSchemaUri), + new JProperty(CatalogConstants.Catalog, CatalogConstants.NuGetCatalogSchemaUri), + new JProperty(CatalogConstants.Xsd, CatalogConstants.XmlSchemaUri), + new JProperty(CatalogConstants.Dependencies, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Dependency), + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.DependencyGroups, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.DependencyGroup), + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.PackageEntries, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.PackageEntryUncapitalized), + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.PackageTypes, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.PackageTypeUncapitalized), + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.SupportedFrameworks, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.SupportedFramework), + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.Tags, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Tag), + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.Vulnerabilities, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Vulnerability), + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.Published, + new JObject(new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.Created, + new JObject(new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.LastEdited, + new JObject(new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.CatalogCommitTimeStamp, + new JObject(new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.Reasons, + new JObject( + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword)))); + + Assert.Equal(expectedContext.ToString(), packageDetails[CatalogConstants.ContextKeyword].ToString()); + } + + private void VerifyCatalogPackageDelete( + PackageDeletion packageOperation, + Uri packageDeleteUri, + JObject packageDelete) + { + var properties = packageDelete.Properties(); + + Assert.Equal(9, properties.Count()); + Assert.Equal(packageDeleteUri.AbsoluteUri, packageDelete[CatalogConstants.IdKeyword].Value()); + Assert.Equal( + new JArray(CatalogConstants.PackageDelete, CatalogConstants.CatalogPermalink), + packageDelete[CatalogConstants.TypeKeyword]); + Assert.Equal(packageOperation.CommitId, packageDelete[CatalogConstants.CatalogCommitId].Value()); + Assert.Equal(packageOperation.CommitTimeStamp, packageDelete[CatalogConstants.CatalogCommitTimeStamp].Value()); + Assert.Equal(packageOperation.PackageIdentity.Id, packageDelete[CatalogConstants.Id].Value()); + Assert.Equal(packageOperation.PackageIdentity.Id, packageDelete[CatalogConstants.OriginalId].Value()); + Assert.Equal( + packageOperation.Published.ToString("O"), + DateTime.Parse(packageDelete[CatalogConstants.Published].Value(), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind).ToString("O")); + Assert.Equal(packageOperation.PackageIdentity.Version.ToNormalizedString(), packageDelete[CatalogConstants.Version].Value()); + + var expectedContext = new JObject( + new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetSchemaUri), + new JProperty(CatalogConstants.Catalog, CatalogConstants.NuGetCatalogSchemaUri), + new JProperty(CatalogConstants.Xsd, CatalogConstants.XmlSchemaUri), + new JProperty(CatalogConstants.Details, CatalogConstants.CatalogDetails), + new JProperty(CatalogConstants.CatalogCommitTimeStamp, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.Published, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.Categories, + new JObject( + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.Entries, + new JObject( + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.Links, + new JObject( + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.Tags, + new JObject( + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.PackageContent, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword)))); + + Assert.Equal(expectedContext.ToString(), packageDelete[CatalogConstants.ContextKeyword].ToString()); + } + + private Uri GetPackageDetailsUri(DateTime catalogTimeStamp, PackageOperation packageOperation) + { + var packageId = packageOperation.PackageIdentity.Id.ToLowerInvariant(); + var packageVersion = packageOperation.PackageIdentity.Version.ToNormalizedString().ToLowerInvariant(); + + return new Uri($"{_baseUri.AbsoluteUri}data/{catalogTimeStamp.ToString(CatalogConstants.UrlTimeStampFormat)}/{packageId}.{packageVersion}.json"); + } + + private DateTime GetUniqueDateTime() + { + var now = DateTimeOffset.UtcNow; + + if (_timestamp == now) + { + _timestamp = now.AddMilliseconds(1); + } + else + { + _timestamp = now; + } + + return _timestamp.UtcDateTime; + } + + private void PublishCreatedPackages(Mock galleryDatabaseHelperMock, int top) + { + var packageOperations = _packageOperations.OfType(); + + foreach (var packageOperation in packageOperations) + { + var createdPackages = new SortedList>(); + createdPackages.Add(packageOperation.FeedPackageDetails.CreatedDate, new List { packageOperation.FeedPackageDetails }); + + galleryDatabaseHelperMock + .Setup(m => m.GetPackagesCreatedSince(_feedLastCreated, top)) + .ReturnsAsync(createdPackages); + + RegisterPackageContentUri(packageOperation.Package); + + _feedLastCreated = packageOperation.FeedPackageDetails.CreatedDate; + } + + galleryDatabaseHelperMock + .Setup(m => m.GetPackagesCreatedSince(_feedLastCreated, top)) + .ReturnsAsync(new SortedList>()); + } + + private void PublishEditedPackages(Mock galleryDatabaseHelperMock, int top) + { + var packageOperations = _packageOperations.OfType() + .Where(entry => entry.FeedPackageDetails.LastEditedDate != DateTime.MinValue); + + foreach (var packageOperation in packageOperations) + { + var editedPackages = new SortedList>(); + editedPackages.Add(packageOperation.FeedPackageDetails.LastEditedDate, new List { packageOperation.FeedPackageDetails }); + + galleryDatabaseHelperMock + .Setup(m => m.GetPackagesEditedSince(_feedLastEdited, top)) + .ReturnsAsync(editedPackages); + + RegisterPackageContentUri(packageOperation.Package); + + _feedLastEdited = packageOperation.FeedPackageDetails.LastEditedDate; + } + + galleryDatabaseHelperMock + .Setup(m => m.GetPackagesEditedSince(_feedLastEdited, top)) + .ReturnsAsync(new SortedList>()); + } + + private TestPackage AddPackageEntry(TestPackage package) + { + var stream = new MemoryStream(); + + package.Stream.Position = 0; + package.Stream.CopyTo(stream); + + stream.Position = 0; + + using (var zip = new ZipArchive(stream, ZipArchiveMode.Update, leaveOpen: true)) + { + var entryName = $"file{zip.Entries.Count - 1}.bin"; + var entry = zip.CreateEntry(entryName); + + using (var entryStream = entry.Open()) + using (var writer = new StreamWriter(entryStream)) + using (var rng = RandomNumberGenerator.Create()) + { + var byteCount = _random.Next(1, 10); + var bytes = new byte[byteCount]; + + rng.GetNonZeroBytes(bytes); + + writer.Write(bytes); + } + } + + return new TestPackage(package.Id, package.Version, package.Author, package.Description, package.Nuspec, stream); + } + + private static JObject ReadJsonWithoutDateTimeHandling(JObject jObject) + { + using (var stringReader = new StringReader(jObject.ToString())) + using (var jsonReader = new JsonTextReader(stringReader)) + { + jsonReader.DateParseHandling = DateParseHandling.None; + + return JToken.ReadFrom(jsonReader) as JObject; + } + } + + private static IReadOnlyList GetPackageEntries(TestPackage package) + { + using (var zip = new ZipArchive(package.Stream, ZipArchiveMode.Read, leaveOpen: true)) + { + return zip.Entries.Select(entry => new PackageEntry(entry)).ToArray(); + } + } + + private static string GetPackageHash(TestPackage package) + { + using (var hashAlgorithm = HashAlgorithm.Create(Constants.Sha512)) + { + package.Stream.Position = 0; + + var hash = hashAlgorithm.ComputeHash(package.Stream); + + package.Stream.Position = 0; + + return Convert.ToBase64String(hash); + } + } + + private static Task GetStreamContentActionAsync(Stream stream) + { + // Ensure we reset the stream position to 0 (a package edit might have happened). + stream.Position = 0; + + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StreamContent(stream) + }; + + response.Content.Headers.Add("Content-Length", stream.Length.ToString()); + + return Task.FromResult(response); + } + + private static Task GetStreamContentActionAsync(string filePath) + { + return GetStreamContentActionAsync(File.OpenRead(filePath)); + } + + private static void VerifyContext(JObject indexOrPage) + { + var expectedContext = new JObject( + new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetCatalogSchemaUri), + new JProperty(CatalogConstants.NuGet, CatalogConstants.NuGetSchemaUri), + new JProperty(CatalogConstants.Items, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Item), + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.Parent, + new JObject(new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), + new JProperty(CatalogConstants.CommitTimeStamp, + new JObject(new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XmlDateTimeSchemaUri))), + new JProperty(CatalogConstants.NuGetLastCreated, + new JObject(new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XmlDateTimeSchemaUri))), + new JProperty(CatalogConstants.NuGetLastEdited, + new JObject(new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XmlDateTimeSchemaUri))), + new JProperty(CatalogConstants.NuGetLastDeleted, + new JObject(new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XmlDateTimeSchemaUri)))); + + Assert.Equal(expectedContext.ToString(), indexOrPage[CatalogConstants.ContextKeyword].ToString()); + } + + private abstract class PackageOperation + { + internal string CommitId { get; set; } + internal string CommitTimeStamp { get; set; } + internal DateTime CommitTimeStampDateTime { get; set; } + internal abstract PackageIdentity PackageIdentity { get; } + } + + private sealed class PackageCreationOrEdit : PackageOperation + { + internal FeedPackageDetails FeedPackageDetails { get; } + internal TestPackage Package { get; } + internal override PackageIdentity PackageIdentity { get; } + + internal PackageCreationOrEdit(TestPackage package, FeedPackageDetails feedPackageDetails) + { + Package = package; + FeedPackageDetails = feedPackageDetails; + PackageIdentity = new PackageIdentity(package.Id, package.Version); + } + } + + private sealed class PackageDeletion : PackageOperation + { + internal DateTimeOffset DeletionTime { get; } + internal JObject Json { get; } + internal override PackageIdentity PackageIdentity { get; } + internal DateTime Published => DeletionTime.UtcDateTime; + internal Uri Uri { get; } + + internal PackageDeletion(Uri uri, JObject json, DateTimeOffset deletionTime) + { + Uri = uri; + Json = json; + DeletionTime = deletionTime; + + var record = json["record"]; + + PackageIdentity = new PackageIdentity( + record["id"].Value(), + new NuGetVersion(record["version"].Value())); + } + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/InMemoryHttpHandler.cs b/tests/NgTests/Infrastructure/InMemoryHttpHandler.cs new file mode 100644 index 000000000..42ec0b11d --- /dev/null +++ b/tests/NgTests/Infrastructure/InMemoryHttpHandler.cs @@ -0,0 +1,40 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace NgTests.Infrastructure +{ + public class InMemoryHttpHandler : HttpMessageHandler + { + private readonly IReadOnlyDictionary _responses; + + public InMemoryHttpHandler(IReadOnlyDictionary responses) + { + _responses = responses; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + HttpResponseMessage responseMessage; + string response; + if (_responses.TryGetValue(request.RequestUri.ToString(), out response)) + { + responseMessage = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(response) + }; + } + else + { + responseMessage = new HttpResponseMessage(HttpStatusCode.NotFound); + } + + return Task.FromResult(responseMessage); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/MemoryStorage.cs b/tests/NgTests/Infrastructure/MemoryStorage.cs new file mode 100644 index 000000000..14371013c --- /dev/null +++ b/tests/NgTests/Infrastructure/MemoryStorage.cs @@ -0,0 +1,195 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace NgTests.Infrastructure +{ + public class MemoryStorage : Storage + { + public ConcurrentDictionary Content { get; } + + public ConcurrentDictionary ContentBytes { get; } + + public ConcurrentDictionary ListMock { get; } + + public MemoryStorage() + : this(new Uri("http://tempuri.org")) + { + } + + public MemoryStorage(Uri baseAddress) + : base(baseAddress) + { + Content = new ConcurrentDictionary(); + ContentBytes = new ConcurrentDictionary(); + ListMock = new ConcurrentDictionary(); + } + + protected MemoryStorage( + Uri baseAddress, + ConcurrentDictionary content, + ConcurrentDictionary contentBytes) + : base(baseAddress) + { + Content = content; + ContentBytes = contentBytes; + ListMock = new ConcurrentDictionary(); + + foreach (var resourceUri in Content.Keys) + { + ListMock[resourceUri] = CreateStorageListItem(resourceUri); + } + } + + public override Task GetOptimisticConcurrencyControlTokenAsync(Uri resourceUri, CancellationToken cancellationToken) + { + return Task.FromResult(OptimisticConcurrencyControlToken.Null); + } + + private static StorageListItem CreateStorageListItem(Uri uri) + { + return new StorageListItem(uri, DateTime.UtcNow); + } + + public virtual Storage WithName(string name) + { + return new MemoryStorage(new Uri(BaseAddress + name), Content, ContentBytes); + } + + protected override Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + protected override async Task OnSaveAsync(Uri resourceUri, StorageContent content, CancellationToken cancellationToken) + { + if (content is StringStorageContentWithAccessCondition accessConditionContent) + { + // Verify the access condition of this request. + var accessCondition = accessConditionContent.AccessCondition; + AssertAccessCondition(resourceUri, accessCondition); + } + + if (content is StringStorageContent stringStorageContent && !(content is StringStorageContentWithETag)) + { + // Give this content an ETag + content = new StringStorageContentWithETag( + stringStorageContent.Content, + Guid.NewGuid().ToString(), + stringStorageContent.ContentType, + stringStorageContent.CacheControl); + } + + Content[resourceUri] = content; + + using (var memoryStream = new MemoryStream()) + { + var contentStream = content.GetContentStream(); + await contentStream.CopyToAsync(memoryStream); + + if (contentStream.CanSeek) + { + contentStream.Position = 0; + } + + ContentBytes[resourceUri] = memoryStream.ToArray(); + } + + ListMock[resourceUri] = CreateStorageListItem(resourceUri); + } + + protected override Task OnLoadAsync(Uri resourceUri, CancellationToken cancellationToken) + { + Content.TryGetValue(resourceUri, out StorageContent content); + + return Task.FromResult(content); + } + + protected override Task OnDeleteAsync(Uri resourceUri, DeleteRequestOptions deleteRequestOptions, CancellationToken cancellationToken) + { + if (deleteRequestOptions is DeleteRequestOptionsWithAccessCondition deleteRequestOptionsWithAccessCondition) + { + // Verify the access condition of this request. + var accessCondition = deleteRequestOptionsWithAccessCondition.AccessCondition; + AssertAccessCondition(resourceUri, accessCondition); + } + + Content.TryRemove(resourceUri, out _); + ContentBytes.TryRemove(resourceUri, out _); + ListMock.TryRemove(resourceUri, out _); + + return Task.FromResult(true); + } + + public override bool Exists(string fileName) + { + return Content.Keys.Any(k => k.PathAndQuery.EndsWith(fileName)); + } + + public override Task> ListAsync(CancellationToken cancellationToken) + { + return Task.FromResult(Content.Keys.AsEnumerable().Select(x => + ListMock.ContainsKey(x) ? ListMock[x] : new StorageListItem(x, DateTime.UtcNow))); + } + + private void AssertAccessCondition(Uri resourceUri, AccessCondition accessCondition) + { + Content.TryGetValue(resourceUri, out var existingContent); + if (IsAccessCondition(AccessCondition.GenerateEmptyCondition(), accessCondition)) + { + return; + } + + if (IsAccessCondition(AccessCondition.GenerateIfNotExistsCondition(), accessCondition)) + { + Assert.Null(existingContent); + return; + } + + if (IsAccessCondition(AccessCondition.GenerateIfExistsCondition(), accessCondition)) + { + Assert.NotNull(existingContent); + return; + } + + if (existingContent is StringStorageContentWithETag eTagContent) + { + var eTag = eTagContent.ETag; + if (IsAccessCondition(AccessCondition.GenerateIfMatchCondition(eTag), accessCondition)) + { + return; + } + } + + throw new InvalidOperationException("Could not validate access condition!"); + } + + private static bool IsAccessCondition(AccessCondition expected, AccessCondition actual) + { + try + { + PackageMonitoringStatusTestUtility.AssertAccessCondition(expected, actual); + return true; + } + catch + { + return false; + } + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/MemoryStorageFactory.cs b/tests/NgTests/Infrastructure/MemoryStorageFactory.cs new file mode 100644 index 000000000..92192f0ec --- /dev/null +++ b/tests/NgTests/Infrastructure/MemoryStorageFactory.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NgTests.Infrastructure +{ + public class MemoryStorageFactory : StorageFactory + { + private readonly Dictionary _cachedStorages; + + public MemoryStorageFactory() + : this(new Uri("https://nuget.test")) + { + } + + public MemoryStorageFactory(Uri baseAddress) + { + BaseAddress = baseAddress ?? throw new ArgumentNullException(nameof(baseAddress)); + + _cachedStorages = new Dictionary(); + } + + public override Storage Create(string name = null) + { + if (!_cachedStorages.ContainsKey(name)) + { + _cachedStorages[name] = (MemoryStorage)new MemoryStorage(BaseAddress).WithName(name); + } + + return _cachedStorages[name]; + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/MockServerHttpClientHandler.cs b/tests/NgTests/Infrastructure/MockServerHttpClientHandler.cs new file mode 100644 index 000000000..657dacf25 --- /dev/null +++ b/tests/NgTests/Infrastructure/MockServerHttpClientHandler.cs @@ -0,0 +1,67 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace NgTests.Infrastructure +{ + public class MockServerHttpClientHandler + : HttpClientHandler + { + public Dictionary>> Actions { get; private set; } + public bool Return404OnUnknownAction { get; set; } + + private readonly object _requestsLock = new object(); + private readonly List _requests = new List(); + + public MockServerHttpClientHandler() + { + Actions = new Dictionary>>(); + Return404OnUnknownAction = true; + } + + public void SetAction(string relativeUrl, Func> action) + { + Actions[relativeUrl] = action; + } + + public IReadOnlyList Requests => _requests; + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + lock (_requestsLock) + { + _requests.Add(request); + } + + Func> action; + + // try with full URL + if (Actions.TryGetValue(request.RequestUri.PathAndQuery, out action)) + { + return await action(request); + } + + // try with full URL ignoring query string + if (Actions.TryGetValue(request.RequestUri.AbsolutePath, out action)) + { + return await action(request); + } + + if (Return404OnUnknownAction) + { + return new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent("Could not find " + request.RequestUri) + }; + } + + return await base.SendAsync(request, cancellationToken); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/MockServerHttpClientHandlerExtensions.cs b/tests/NgTests/Infrastructure/MockServerHttpClientHandlerExtensions.cs new file mode 100644 index 000000000..f5eebdc4e --- /dev/null +++ b/tests/NgTests/Infrastructure/MockServerHttpClientHandlerExtensions.cs @@ -0,0 +1,47 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NgTests.Infrastructure +{ + public static class MockServerHttpClientHandlerExtensions + { + public static async Task AddStorageAsync(this MockServerHttpClientHandler handler, IStorage storage) + { + var files = (await storage.ListAsync(CancellationToken.None)).Select(x => x.Uri); + + foreach (var file in files) + { + var storageFileUrl = file; + var relativeFileUrl = "/" + storageFileUrl.ToString().Replace(storage.BaseAddress.ToString(), string.Empty); + + handler.SetAction(relativeFileUrl, async message => + { + var content = await storage.LoadAsync(storageFileUrl, CancellationToken.None); + + var response = new HttpResponseMessage(HttpStatusCode.OK); + + if (!string.IsNullOrEmpty(content.CacheControl)) + { + response.Headers.CacheControl = CacheControlHeaderValue.Parse(content.CacheControl); + } + response.Content = new StreamContent(content.GetContentStream()); + + if (!string.IsNullOrEmpty(content.ContentType)) + { + response.Content.Headers.ContentType = new MediaTypeHeaderValue(content.ContentType); + } + + return response; + }); + } + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/MockTelemetryService.cs b/tests/NgTests/Infrastructure/MockTelemetryService.cs new file mode 100644 index 000000000..8d0ffc108 --- /dev/null +++ b/tests/NgTests/Infrastructure/MockTelemetryService.cs @@ -0,0 +1,138 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Moq; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Versioning; + +namespace NgTests.Infrastructure +{ + public sealed class MockTelemetryService : ITelemetryService + { + private readonly List _trackDurationCalls = new List(); + private readonly List _trackMetricCalls = new List(); + private readonly object _durationCallsSyncObject = new object(); + private readonly object _metricCallsSyncObject = new object(); + + public IDictionary GlobalDimensions => throw new NotImplementedException(); + + public IReadOnlyList TrackDurationCalls => _trackDurationCalls; + public IReadOnlyList TrackMetricCalls => _trackMetricCalls; + + public void TrackCatalogIndexReadDuration(TimeSpan duration, Uri uri) + { + } + + public void TrackCatalogIndexWriteDuration(TimeSpan duration, Uri uri) + { + } + + public IDisposable TrackIndexCommitDuration() + { + return TrackDuration(nameof(TrackIndexCommitDuration)); + } + + public void TrackIndexCommitTimeout() + { + } + + public void TrackHandlerFailedToProcessPackage(IPackagesContainerHandler handler, string packageId, NuGetVersion packageVersion) + { + } + + public void TrackPackageMissingHash(string packageId, NuGetVersion packageVersion) + { + } + + public void TrackPackageHasIncorrectHash(string packageId, NuGetVersion packageVersion) + { + } + + public void TrackPackageAlreadyHasHash(string packageId, NuGetVersion packageVersion) + { + } + + public void TrackPackageHashFixed(string packageId, NuGetVersion packageVersion) + { + } + + public DurationMetric TrackDuration(string name, IDictionary properties = null) + { + lock (_durationCallsSyncObject) + { + _trackDurationCalls.Add(new TelemetryCall(name, properties)); + } + + return new DurationMetric(Mock.Of(), name, properties); + } + + public void TrackMetric(string name, ulong metric, IDictionary properties = null) + { + lock (_metricCallsSyncObject) + { + _trackMetricCalls.Add(new TrackMetricCall(name, metric, properties)); + } + } + + public void TrackIconExtractionFailure(string packageId, string normalizedPackageVersion) + { + } + + public IDisposable TrackGetPackageDetailsQueryDuration(Db2CatalogCursor cursor) + { + var properties = new Dictionary() + { + { TelemetryConstants.Method, cursor.ColumnName }, + { TelemetryConstants.BatchItemCount, cursor.Top.ToString() }, + { TelemetryConstants.CursorValue, cursor.CursorValue.ToString("O") } + }; + + return TrackDuration(nameof(TrackGetPackageDetailsQueryDuration), properties); + } + + public IDisposable TrackGetPackageQueryDuration(string packageId, string packageVersion) + { + var properties = new Dictionary() + { + { TelemetryConstants.Id, packageId }, + { TelemetryConstants.Version, packageVersion } + }; + + return TrackDuration(nameof(TrackGetPackageQueryDuration), properties); + } + + public void TrackExternalIconIngestionSuccess(string packageId, string normalizedPackageVersion) + { + } + + public void TrackIconExtractionSuccess(string packageId, string normalizedPackageVersion) + { + } + + public IDisposable TrackExternalIconProcessingDuration(string packageId, string normalizedPackageVersion) + { + return TrackDuration(nameof(TrackExternalIconProcessingDuration)); + } + + public IDisposable TrackEmbeddedIconProcessingDuration(string packageId, string normalizedPackageVersion) + { + return TrackDuration(nameof(TrackEmbeddedIconProcessingDuration)); + } + + public void TrackIconDeletionSuccess(string packageId, string normalizedPackageVersion) + { + } + + public void TrackIconDeletionFailure(string packageId, string normalizedPackageVersion) + { + } + + public void TrackExternalIconIngestionFailure(string packageId, string normalizedPackageVersion) + { + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/NoRetryStrategy.cs b/tests/NgTests/Infrastructure/NoRetryStrategy.cs new file mode 100644 index 000000000..78f1c6b91 --- /dev/null +++ b/tests/NgTests/Infrastructure/NoRetryStrategy.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog; + +namespace NgTests.Infrastructure +{ + /// + /// Simple no-retry passthrough to avoid the full default exponential retry strategy in unit tests. + /// + public sealed class NoRetryStrategy : IHttpRetryStrategy + { + public Task SendAsync(HttpClient client, Uri address, CancellationToken cancellationToken) + { + return client.GetAsync(address, cancellationToken); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/ODataFeedHelper.cs b/tests/NgTests/Infrastructure/ODataFeedHelper.cs new file mode 100644 index 000000000..883226e0a --- /dev/null +++ b/tests/NgTests/Infrastructure/ODataFeedHelper.cs @@ -0,0 +1,79 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Xml.Linq; +using NuGet.Services.Metadata.Catalog.Helpers; + +namespace NgTests.Infrastructure +{ + public static class ODataFeedHelper + { + public static string ToODataFeed(IEnumerable packages, Uri baseUri, string title) + { + string nsAtom = "http://www.w3.org/2005/Atom"; + var id = string.Format(CultureInfo.InvariantCulture, "{0}{1}", baseUri, title); + XDocument doc = new XDocument( + new XElement(XName.Get("feed", nsAtom), + new XElement(XName.Get("id", nsAtom), id), + new XElement(XName.Get("title", nsAtom), title))); + + foreach (var package in packages) + { + doc.Root.Add(ToODataEntryXElement(package, baseUri)); + } + + return doc.ToString(); + } + + private static XElement ToODataEntryXElement(ODataPackage package, Uri baseUri) + { + string nsAtom = "http://www.w3.org/2005/Atom"; + XNamespace nsDataService = "http://schemas.microsoft.com/ado/2007/08/dataservices"; + string nsMetadata = "http://schemas.microsoft.com/ado/2007/08/dataservices/metadata"; + string downloadUrl = string.Format( + CultureInfo.InvariantCulture, + "{0}package/{1}/{2}", baseUri, package.Id, NuGetVersionUtility.NormalizeVersion(package.Version)); + string entryId = string.Format( + CultureInfo.InvariantCulture, + "{0}Packages(Id='{1}',Version='{2}')", + baseUri, package.Id, NuGetVersionUtility.NormalizeVersion(package.Version)); + + const string FeedDateTimeFormat = "yyyy-MM-ddTHH:mm:ss.FFF"; + + var entry = new XElement(XName.Get("entry", nsAtom), + new XAttribute(XNamespace.Xmlns + "d", nsDataService.ToString()), + new XAttribute(XNamespace.Xmlns + "m", nsMetadata), + new XElement(XName.Get("id", nsAtom), entryId), + new XElement(XName.Get("title", nsAtom), package.Id), + new XElement(XName.Get("content", nsAtom), + new XAttribute("type", "application/zip"), + new XAttribute("src", downloadUrl)), + new XElement(XName.Get("properties", nsMetadata), + new XElement(nsDataService + "Id", package.Id), + new XElement(nsDataService + "NormalizedVersion", package.Version), + new XElement(nsDataService + "Version", package.Version), + new XElement(nsDataService + "PackageHash", "dummy"), + new XElement(nsDataService + "PackageHashAlgorithm", "dummy"), + new XElement(nsDataService + "Description", package.Description), + new XElement(nsDataService + "Listed", package.Listed), + + + new XElement(nsDataService + "Created", package.Created.ToString(FeedDateTimeFormat)), + new XElement(nsDataService + "LastEdited", package.LastEdited?.ToString(FeedDateTimeFormat)), + new XElement(nsDataService + "Published", package.Published.ToString(FeedDateTimeFormat)), + new XElement(nsDataService + "LicenseNames", package.LicenseNames), + new XElement(nsDataService + "LicenseReportUrl", package.LicenseReportUrl))); + + return entry; + } + + public static string ToOData(ODataPackage package, Uri baseUri) + { + XDocument doc = new XDocument(ToODataEntryXElement(package, baseUri)); + return doc.ToString(); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/ODataPackage.cs b/tests/NgTests/Infrastructure/ODataPackage.cs new file mode 100644 index 000000000..eec00b72c --- /dev/null +++ b/tests/NgTests/Infrastructure/ODataPackage.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace NgTests.Infrastructure +{ + public class ODataPackage + { + public string Id { get; set; } + public string Version { get; set; } + public string Description { get; set; } + public string Hash { get; set; } + public bool Listed { get; set; } + + public DateTime Created { get; set; } + public DateTime? LastEdited { get; set; } + public DateTime Published { get; set; } + public string LicenseNames { get; set; } + public string LicenseReportUrl { get; set; } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/StorageContentExtensions.cs b/tests/NgTests/Infrastructure/StorageContentExtensions.cs new file mode 100644 index 000000000..6d6e5024b --- /dev/null +++ b/tests/NgTests/Infrastructure/StorageContentExtensions.cs @@ -0,0 +1,25 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.IO; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NgTests.Infrastructure +{ + public static class StorageContentExtensions + { + public static string GetContentString(this StorageContent content) + { + var stringStorageContent = content as StringStorageContent; + if (stringStorageContent != null) + { + return stringStorageContent.Content; + } + + using (var reader = new StreamReader(content.GetContentStream())) + { + return reader.ReadToEnd(); + } + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/TelemetryCall.cs b/tests/NgTests/Infrastructure/TelemetryCall.cs new file mode 100644 index 000000000..c45f8f30f --- /dev/null +++ b/tests/NgTests/Infrastructure/TelemetryCall.cs @@ -0,0 +1,24 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Collections.ObjectModel; + +namespace NgTests.Infrastructure +{ + public class TelemetryCall + { + public string Name { get; } + public IReadOnlyDictionary Properties { get; } + + internal TelemetryCall(string name, IDictionary properties) + { + Name = name; + + if (properties != null) + { + Properties = new ReadOnlyDictionary(properties); + } + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/TestDirectory.cs b/tests/NgTests/Infrastructure/TestDirectory.cs new file mode 100644 index 000000000..73229f5bd --- /dev/null +++ b/tests/NgTests/Infrastructure/TestDirectory.cs @@ -0,0 +1,58 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; + +namespace NgTests.Infrastructure +{ + public sealed class TestDirectory : IDisposable + { + public string FullPath { get; } + + private TestDirectory(string fullPath) + { + FullPath = fullPath; + } + + public void Dispose() + { + DeleteDirectory(FullPath); + } + + public static TestDirectory Create() + { + var baseDirectoryPath = Path.Combine(Path.GetTempPath(), "NuGetTestFolder"); + var subdirectoryName = Guid.NewGuid().ToString(); + var fullPath = Path.Combine(baseDirectoryPath, subdirectoryName); + + Directory.CreateDirectory(fullPath); + + return new TestDirectory(fullPath); + } + + public static implicit operator string(TestDirectory directory) + { + return directory.FullPath; + } + + public override string ToString() + { + return FullPath; + } + + private static void DeleteDirectory(string path) + { + if (Directory.Exists(path)) + { + try + { + Directory.Delete(path, recursive: true); + } + catch + { + } + } + } + } +} diff --git a/tests/NgTests/Infrastructure/TestLogger.cs b/tests/NgTests/Infrastructure/TestLogger.cs new file mode 100644 index 000000000..c5d9c14e6 --- /dev/null +++ b/tests/NgTests/Infrastructure/TestLogger.cs @@ -0,0 +1,51 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; + +namespace NgTests.Infrastructure +{ + internal class TestLogger : ILogger + { + private readonly ITestOutputHelper _testOutputHelper; + + public TestLogger(ITestOutputHelper testOutputHelper) + { + _testOutputHelper = testOutputHelper ?? throw new ArgumentNullException(nameof(testOutputHelper)); + } + + public void Log(LogLevel logLevel, int eventId, object state, Exception exception, Func formatter) + { + _testOutputHelper.WriteLine($"{logLevel}: {formatter(state, exception)}"); + } + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + _testOutputHelper.WriteLine($"{logLevel}: {formatter(state, exception)}"); + } + + public bool IsEnabled(LogLevel logLevel) + { + return true; + } + + public IDisposable BeginScope(TState state) + { + return BeginScopeImpl(state); + } + + public IDisposable BeginScopeImpl(object state) + { + return new TestLoggerScoper(); + } + + private class TestLoggerScoper : IDisposable + { + public void Dispose() + { + } + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/TestLoggerFactory.cs b/tests/NgTests/Infrastructure/TestLoggerFactory.cs new file mode 100644 index 000000000..10962f670 --- /dev/null +++ b/tests/NgTests/Infrastructure/TestLoggerFactory.cs @@ -0,0 +1,35 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; + +namespace NgTests.Infrastructure +{ + internal class TestLoggerFactory + : ILoggerFactory + { + private readonly ITestOutputHelper _testOutputHelper; + + public TestLoggerFactory(ITestOutputHelper testOutputHelper) + { + _testOutputHelper = testOutputHelper ?? throw new ArgumentNullException(nameof(testOutputHelper)); + } + + public void Dispose() + { + } + + public ILogger CreateLogger(string categoryName) + { + return new TestLogger(_testOutputHelper); + } + + public void AddProvider(ILoggerProvider provider) + { + } + + public LogLevel MinimumLevel { get; set; } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/TestPackage.cs b/tests/NgTests/Infrastructure/TestPackage.cs new file mode 100644 index 000000000..70e8270ab --- /dev/null +++ b/tests/NgTests/Infrastructure/TestPackage.cs @@ -0,0 +1,122 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.IO.Compression; +using System.Security.Cryptography; +using NuGet.Versioning; + +namespace NgTests.Infrastructure +{ + public sealed class TestPackage : IDisposable + { + private bool _isDisposed; + + public string Id { get; } + public NuGetVersion Version { get; } + public string Author { get; } + public string Description { get; } + public string Nuspec { get; } + public Stream Stream { get; } + + public TestPackage(string id, NuGetVersion version, string author, string description, string nuspec, Stream stream) + { + Id = id; + Version = version; + Author = author; + Description = description; + Nuspec = nuspec; + Stream = stream; + } + + public void Dispose() + { + if (!_isDisposed) + { + Stream.Dispose(); + + GC.SuppressFinalize(this); + + _isDisposed = true; + } + } + + public static TestPackage Create(Random random) + { + var id = TestUtility.CreateRandomAlphanumericString(random); + var version = CreateRandomVersion(random); + var author = TestUtility.CreateRandomAlphanumericString(random); + var description = TestUtility.CreateRandomAlphanumericString(random); + var nuspec = CreateNuspec(id, version, author, description); + + using (var rng = RandomNumberGenerator.Create()) + { + var stream = CreatePackageStream(id, nuspec, rng, random); + + return new TestPackage(id, version, author, description, nuspec, stream); + } + } + + private static string CreateNuspec(string id, NuGetVersion version, string author, string description) + { + return $@" + + + {id} + {version.ToNormalizedString()} + {author} + {description} + +"; + } + + private static MemoryStream CreatePackageStream(string id, string nuspec, RandomNumberGenerator rng, Random random) + { + var stream = new MemoryStream(); + + using (var zip = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: true)) + { + var entry = zip.CreateEntry($"{id}.nuspec"); + + using (var entryStream = entry.Open()) + using (var writer = new StreamWriter(entryStream)) + { + writer.Write(nuspec); + } + + // The max value is arbitrary. We just need a little variation in package entries. + var entryCount = random.Next(minValue: 1, maxValue: 3); + + for (var i = 0; i < entryCount; ++i) + { + entry = zip.CreateEntry($"file{i}.bin"); + + using (var entryStream = entry.Open()) + using (var writer = new StreamWriter(entryStream)) + { + var byteCount = random.Next(1, 10); + var bytes = new byte[byteCount]; + + rng.GetNonZeroBytes(bytes); + + writer.Write(bytes); + } + } + } + + stream.Position = 0; + + return stream; + } + + private static NuGetVersion CreateRandomVersion(Random random) + { + var major = random.Next(minValue: 1, maxValue: 10); + var minor = random.Next(minValue: 1, maxValue: 10); + var build = random.Next(minValue: 1, maxValue: 10); + + return new NuGetVersion(major, minor, build); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/TestStorageFactory.cs b/tests/NgTests/Infrastructure/TestStorageFactory.cs new file mode 100644 index 000000000..e471f38bd --- /dev/null +++ b/tests/NgTests/Infrastructure/TestStorageFactory.cs @@ -0,0 +1,31 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NgTests.Infrastructure +{ + public class TestStorageFactory + : StorageFactory + { + private readonly Func _createStorage; + + public TestStorageFactory() + : this(name => new MemoryStorage()) + { + } + + public TestStorageFactory(Func createStorage) + { + _createStorage = createStorage; + + BaseAddress = _createStorage(null).BaseAddress; + } + + public override Storage Create(string name = null) + { + return _createStorage(name); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/TestUtility.cs b/tests/NgTests/Infrastructure/TestUtility.cs new file mode 100644 index 000000000..b90767456 --- /dev/null +++ b/tests/NgTests/Infrastructure/TestUtility.cs @@ -0,0 +1,82 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using Newtonsoft.Json.Linq; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; + +namespace NgTests.Infrastructure +{ + public static class TestUtility + { + private static readonly Random _random = new Random(); + + public static string CreateRandomAlphanumericString() + { + return CreateRandomAlphanumericString(_random); + } + + public static string CreateRandomAlphanumericString(Random random) + { + const string characters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + + return new string( + Enumerable.Repeat(characters, count: 16) + .Select(s => s[random.Next(s.Length)]) + .ToArray()); + } + + public static JObject CreateCatalogContextJObject() + { + return new JObject( + new JProperty(CatalogConstants.VocabKeyword, CatalogConstants.NuGetSchemaUri), + new JProperty(CatalogConstants.NuGet, CatalogConstants.NuGetSchemaUri), + new JProperty(CatalogConstants.Items, + new JObject( + new JProperty(CatalogConstants.IdKeyword, CatalogConstants.Item), + new JProperty(CatalogConstants.ContainerKeyword, CatalogConstants.SetKeyword))), + new JProperty(CatalogConstants.Parent, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.IdKeyword))), + new JProperty(CatalogConstants.CommitTimeStamp, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.NuGetLastCreated, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.NuGetLastEdited, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime))), + new JProperty(CatalogConstants.NuGetLastDeleted, + new JObject( + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.XsdDateTime)))); + } + + public static JObject CreateCatalogCommitItemJObject( + DateTime commitTimeStamp, + PackageIdentity packageIdentity, + string commitId = null) + { + return new JObject( + new JProperty(CatalogConstants.IdKeyword, $"https://nuget.test/{packageIdentity.Id}"), + new JProperty(CatalogConstants.TypeKeyword, CatalogConstants.NuGetPackageDetails), + new JProperty(CatalogConstants.CommitTimeStamp, commitTimeStamp.ToString("O")), + new JProperty(CatalogConstants.CommitId, commitId ?? Guid.NewGuid().ToString()), + new JProperty(CatalogConstants.NuGetId, packageIdentity.Id), + new JProperty(CatalogConstants.NuGetVersion, packageIdentity.Version.ToNormalizedString())); + } + + public static CatalogCommitItem CreateCatalogCommitItem( + DateTime commitTimeStamp, + PackageIdentity packageIdentity, + string commitId = null) + { + var context = CreateCatalogContextJObject(); + var commitItem = CreateCatalogCommitItemJObject(commitTimeStamp, packageIdentity, commitId); + + return CatalogCommitItem.Create(context, commitItem); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Infrastructure/TrackMetricCall.cs b/tests/NgTests/Infrastructure/TrackMetricCall.cs new file mode 100644 index 000000000..a98162aa9 --- /dev/null +++ b/tests/NgTests/Infrastructure/TrackMetricCall.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; + +namespace NgTests.Infrastructure +{ + public sealed class TrackMetricCall : TelemetryCall + { + public ulong Metric { get; } + + internal TrackMetricCall(string name, ulong metric, IDictionary properties) + : base(name, properties) + { + Metric = metric; + } + } +} \ No newline at end of file diff --git a/tests/NgTests/NgJobFactoryTests.cs b/tests/NgTests/NgJobFactoryTests.cs new file mode 100644 index 000000000..9008158d9 --- /dev/null +++ b/tests/NgTests/NgJobFactoryTests.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using Moq; +using Ng; +using NuGet.Services.Logging; +using Xunit; + +namespace NgTests +{ + public class NgJobFactoryTests + { + [Theory] + [MemberData(nameof(CanActivateJobsData))] + public void CanActivateJobs(string name, Type type) + { + // Arrange + var telemetryClient = new Mock(); + var loggerFactory = new Mock(); + loggerFactory + .Setup(x => x.CreateLogger(It.IsAny())) + .Returns(() => new Mock().Object); + + // Act + var job = NgJobFactory.GetJob(name, loggerFactory.Object, telemetryClient.Object, new Dictionary()); + + // Assert + Assert.NotNull(job); + Assert.IsType(type, job); + } + + public static IEnumerable CanActivateJobsData => NgJobFactory + .JobMap + .OrderBy(x => x.Key) + .Select(x => new object[] { x.Key, x.Value }); + } +} diff --git a/tests/NgTests/NgTests.csproj b/tests/NgTests/NgTests.csproj new file mode 100644 index 000000000..ae3e8440d --- /dev/null +++ b/tests/NgTests/NgTests.csproj @@ -0,0 +1,225 @@ + + + + + Debug + AnyCPU + {05C1C78A-9966-4922-9065-A099023E7366} + Library + Properties + NgTests + NgTests + v4.7.2 + 512 + + + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + true + true + + + + + + + + + + + + + + + + + + + + + + + + True + True + TestRegistrationEntries.resx + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + True + True + TestCatalogEntries.resx + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Packages\TestPackage.SemVer2.1.0.0-alpha.1.nupkg + PreserveNewest + + + Always + + + Always + + + Always + + + Always + + + Always + + + Always + + + Always + + + Always + + + + + PublicResXFileCodeGenerator + TestCatalogEntries.Designer.cs + Designer + + + PublicResXFileCodeGenerator + TestRegistrationEntries.Designer.cs + + + + + + + + {e97f23b8-ecb0-4afa-b00c-015c39395fef} + NuGet.Services.Metadata.Catalog + + + {5234d86f-2c0e-4181-aab7-bbda3253b4e1} + Ng + + + {1745a383-d0be-484b-81eb-27b20f6ac6c5} + NuGet.Services.Metadata.Catalog.Monitoring + + + + + 4.10.1 + + + 2.75.0 + + + 2.75.0 + + + 2.4.1 + + + 2.4.1 + runtime; build; native; contentfiles; analyzers + all + + + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + \ No newline at end of file diff --git a/tests/NgTests/PackageFixup/FixPackageHashHandlerFacts.cs b/tests/NgTests/PackageFixup/FixPackageHashHandlerFacts.cs new file mode 100644 index 000000000..145ec7be3 --- /dev/null +++ b/tests/NgTests/PackageFixup/FixPackageHashHandlerFacts.cs @@ -0,0 +1,103 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.WindowsAzure.Storage; +using Moq; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Versioning; +using Xunit; + +namespace NgTests.PackageFixup +{ + public class FixPackageHashHandlerFacts + { + private static readonly PackageIdentity PackageIdentity = new PackageIdentity("TestUnsigned", new NuGetVersion("1.0.0")); + + private readonly CatalogIndexEntry _packageEntry; + private readonly Mock _blob; + + private readonly Mock _telemetryService; + private readonly FixPackageHashHandler _target; + + public FixPackageHashHandlerFacts() + { + var messageHandler = new MockServerHttpClientHandler(); + messageHandler.SetAction( + "/packages/testunsigned.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StreamContent(File.OpenRead("Packages\\TestUnsigned.1.0.0.nupkg")) + } + )); + + _packageEntry = new CatalogIndexEntry( + new Uri("http://localhost/catalog/entry.json"), + "nuget:PackageDetails", + "123", + DateTime.UtcNow, + PackageIdentity); + + _blob = new Mock(); + _blob.Setup(b => b.Uri).Returns(new Uri("http://localhost/packages/testunsigned.1.0.0.nupkg")); + + _telemetryService = new Mock(); + _target = new FixPackageHashHandler( + new HttpClient(messageHandler), + _telemetryService.Object, + Mock.Of>()); + } + + [Fact] + public async Task SkipsPackagesThatAlreadyHaveAHash() + { + // Arrange + _blob.Setup(b => b.ContentMD5).Returns("Incorrect MD5 Content Hash"); + + // Act + await _target.ProcessPackageAsync(_packageEntry, _blob.Object); + + // Assert + _blob.Verify(b => b.FetchAttributesAsync(It.IsAny()), Times.Once); + _telemetryService.Verify(t => t.TrackPackageAlreadyHasHash(PackageIdentity.Id, PackageIdentity.Version), Times.Once); + _telemetryService.Verify(t => t.TrackPackageHashFixed(PackageIdentity.Id, PackageIdentity.Version), Times.Never); + } + + [Fact] + public async Task AddsHashIfPackageIsMissingHash() + { + // Arrange + string hash = null; + _blob.Setup(b => b.ContentMD5).Returns(null); + _blob.SetupSet(b => b.ContentMD5 = It.IsAny()).Callback(h => hash = h); + _blob.Setup(b => b.ETag).Returns("abc"); + + // Act + await _target.ProcessPackageAsync(_packageEntry, _blob.Object); + + // Assert + Assert.Equal("HwmmE4OAMb2Lr3/Yj7oK6w==", hash); + _blob.Verify(b => b.FetchAttributesAsync(It.IsAny()), Times.Once); + _blob.Verify( + b => b.SetPropertiesAsync( + It.Is(c => + c.IfModifiedSinceTime == null && + c.IfMatchETag == "abc"), + null, + null), + Times.Once); + + _telemetryService.Verify(t => t.TrackPackageAlreadyHasHash(PackageIdentity.Id, PackageIdentity.Version), Times.Never); + _telemetryService.Verify(t => t.TrackPackageHashFixed(PackageIdentity.Id, PackageIdentity.Version), Times.Once); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/PackageFixup/PackagesContainerCatalogProcessorFacts.cs b/tests/NgTests/PackageFixup/PackagesContainerCatalogProcessorFacts.cs new file mode 100644 index 000000000..b9dc67666 --- /dev/null +++ b/tests/NgTests/PackageFixup/PackagesContainerCatalogProcessorFacts.cs @@ -0,0 +1,262 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Sockets; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Moq; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Versioning; +using Xunit; + +namespace NgTests.PackageFixup +{ + public class PackagesContainerCatalogProcessorFacts + { + private static readonly PackageIdentity PackageIdentity = new PackageIdentity("TestPackage", new NuGetVersion("1.0.0")); + + private readonly CatalogIndexEntry _catalogEntry; + private readonly Mock _rawBlob; + private readonly Mock _handler; + private readonly Mock _telemetryService; + private readonly PackagesContainerCatalogProcessor _target; + + public PackagesContainerCatalogProcessorFacts() + { + _rawBlob = new Mock(MockBehavior.Strict, new Uri("http://localhost/packages/testpackage.1.0.0.nupkg")); + + var container = new Mock(MockBehavior.Strict, new Uri("http://localhost/packages/")); + container.Setup(c => c.GetBlockBlobReference("testpackage.1.0.0.nupkg")).Returns(_rawBlob.Object); + + _catalogEntry = new CatalogIndexEntry( + new Uri("http://localhost/catalog/entry.json"), + "nuget:PackageDetails", + "123", + DateTime.UtcNow, + PackageIdentity); + + _handler = new Mock(); + _telemetryService = new Mock(); + _target = new PackagesContainerCatalogProcessor( + container.Object, + _handler.Object, + _telemetryService.Object, + Mock.Of>()); + } + + [Fact] + public async Task CallsHandler() + { + // Act + await _target.ProcessCatalogIndexEntryAsync(_catalogEntry); + + // Assert + _handler.Verify( + h => h.ProcessPackageAsync( + _catalogEntry, + It.Is(b => b.Uri == _rawBlob.Object.Uri)), + Times.Once); + + _telemetryService.Verify( + t => t.TrackHandlerFailedToProcessPackage( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task DoesNotRetryIfHandlerThrowsBlobDoesNotExistException() + { + // Arrange + var storageException = new StorageException( + new RequestResult + { + HttpStatusCode = (int)HttpStatusCode.NotFound + }, + "Message", + inner: null); + + _handler.Setup(h => h.ProcessPackageAsync(It.IsAny(), It.IsAny())) + .Throws(storageException); + + // Act + await _target.ProcessCatalogIndexEntryAsync(_catalogEntry); + + // Assert + _handler.Verify( + h => h.ProcessPackageAsync( + _catalogEntry, + It.Is(b => b.Uri == _rawBlob.Object.Uri)), + Times.Once); + + _telemetryService.Verify( + t => t.TrackHandlerFailedToProcessPackage( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task DoesNotRetryIfHandlerThrowsUnknownException() + { + // Arrange + var exception = new Exception("Unknown exception!"); + + _handler.Setup(h => h.ProcessPackageAsync(It.IsAny(), It.IsAny())) + .Throws(exception); + + // Act + await _target.ProcessCatalogIndexEntryAsync(_catalogEntry); + + // Assert + _handler.Verify( + h => h.ProcessPackageAsync( + _catalogEntry, + It.Is(b => b.Uri == _rawBlob.Object.Uri)), + Times.Once); + + _telemetryService.Verify( + t => t.TrackHandlerFailedToProcessPackage( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Theory] + [MemberData(nameof(RetriesIfHandlerThrowsKnownExceptionData))] + public async Task RetriesIfHandlerThrowsKnownException(Exception exception, bool retries) + { + // Arrange + _handler.Setup(h => h.ProcessPackageAsync(It.IsAny(), It.IsAny())) + .Throws(exception); + + // Act + await _target.ProcessCatalogIndexEntryAsync(_catalogEntry); + + // Assert + _handler.Verify( + h => h.ProcessPackageAsync( + _catalogEntry, + It.Is(b => b.Uri == _rawBlob.Object.Uri)), + retries ? Times.Exactly(5) : Times.Once()); + + _telemetryService.Verify( + t => t.TrackHandlerFailedToProcessPackage( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task CanSucceedAfterRetry() + { + // Arrange + var threw = false; + _handler.Setup(h => h.ProcessPackageAsync(It.IsAny(), It.IsAny())) + .Callback(() => + { + if (!threw) + { + threw = true; + throw new TimeoutException(); + } + }) + .Returns(Task.CompletedTask); + + // Act + await _target.ProcessCatalogIndexEntryAsync(_catalogEntry); + + // Assert + _handler.Verify( + h => h.ProcessPackageAsync( + _catalogEntry, + It.Is(b => b.Uri == _rawBlob.Object.Uri)), + Times.Exactly(2)); + + _telemetryService.Verify( + t => t.TrackHandlerFailedToProcessPackage( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + public static IEnumerable RetriesIfHandlerThrowsKnownExceptionData() + { + // Unknown exceptions should not retry. + yield return new object[] + { + new Exception("Unknown exception!"), + false + }; + + // Failed update due to ETag condition exceptions should retry + yield return new object[] + { + new StorageException( + new RequestResult + { + HttpStatusCode = (int)HttpStatusCode.PreconditionFailed + }, + "Message", + inner: null), + true + }; + + // Timeout exceptions should retry. + yield return new object[] + { + new TaskCanceledException(), + true + }; + + yield return new object[] + { + new TimeoutException(), + true + }; + + yield return new object[] + { + new IOException("IO wrapped exception", new TimeoutException()), + true + }; + + yield return new object[] + { + new SocketException(), + true + }; + + yield return new object[] + { + new IOException("IO wrapped exception", new SocketException()), + true + }; + + yield return new object[] + { + new WebException("Timeout", WebExceptionStatus.Timeout), + true + }; + + yield return new object[] + { + new IOException("IO wrapped exception", new WebException("Timeout", WebExceptionStatus.Timeout)), + true + }; + } + } +} diff --git a/tests/NgTests/PackageFixup/ValidatePackageHashHandlerFacts.cs b/tests/NgTests/PackageFixup/ValidatePackageHashHandlerFacts.cs new file mode 100644 index 000000000..27d7284ed --- /dev/null +++ b/tests/NgTests/PackageFixup/ValidatePackageHashHandlerFacts.cs @@ -0,0 +1,104 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Versioning; +using Xunit; + +namespace NgTests.PackageFixup +{ + public class ValidatePackageHashHandlerFacts + { + private static readonly PackageIdentity PackageIdentity = new PackageIdentity("TestUnsigned", new NuGetVersion("1.0.0")); + + private readonly CatalogIndexEntry _packageEntry; + private readonly Mock _blob; + + private readonly Mock _telemetryService; + private readonly ValidatePackageHashHandler _target; + + public ValidatePackageHashHandlerFacts() + { + var messageHandler = new MockServerHttpClientHandler(); + messageHandler.SetAction( + "/packages/testunsigned.1.0.0.nupkg", + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StreamContent(File.OpenRead("Packages\\TestUnsigned.1.0.0.nupkg")) + } + )); + + _packageEntry = new CatalogIndexEntry( + new Uri("http://localhost/catalog/entry.json"), + "nuget:PackageDetails", + "123", + DateTime.UtcNow, + PackageIdentity); + + _blob = new Mock(); + _blob.Setup(b => b.Uri).Returns(new Uri("http://localhost/packages/testunsigned.1.0.0.nupkg")); + + _telemetryService = new Mock(); + _target = new ValidatePackageHashHandler( + new HttpClient(messageHandler), + _telemetryService.Object, + Mock.Of>()); + } + + [Fact] + public async Task ReportsPackagesThatAreMissingAHash() + { + // Arrange + _blob.Setup(b => b.ContentMD5).Returns(null); + + // Act + await _target.ProcessPackageAsync(_packageEntry, _blob.Object); + + // Assert + _blob.Verify(b => b.FetchAttributesAsync(It.IsAny()), Times.Once); + _telemetryService.Verify(t => t.TrackPackageMissingHash(PackageIdentity.Id, PackageIdentity.Version), Times.Once); + _telemetryService.Verify(t => t.TrackPackageHasIncorrectHash(PackageIdentity.Id, PackageIdentity.Version), Times.Never); + } + + [Fact] + public async Task ReportsPackagesWithIncorrectHash() + { + // Arrange + _blob.Setup(b => b.ContentMD5).Returns("Incorrect MD5 Content Hash"); + + // Act + await _target.ProcessPackageAsync(_packageEntry, _blob.Object); + + // Assert + _blob.Verify(b => b.FetchAttributesAsync(It.IsAny()), Times.Once); + _telemetryService.Verify(t => t.TrackPackageMissingHash(PackageIdentity.Id, PackageIdentity.Version), Times.Never); + _telemetryService.Verify(t => t.TrackPackageHasIncorrectHash(PackageIdentity.Id, PackageIdentity.Version), Times.Once); + } + + [Fact] + public async Task ReportsNothingIfPackageHasCorrectHash() + { + // Arrange + _blob.Setup(b => b.ContentMD5).Returns("HwmmE4OAMb2Lr3/Yj7oK6w=="); + + // Act + await _target.ProcessPackageAsync(_packageEntry, _blob.Object); + + // Assert + _blob.Verify(b => b.FetchAttributesAsync(It.IsAny()), Times.Once); + _telemetryService.Verify(t => t.TrackPackageMissingHash(PackageIdentity.Id, PackageIdentity.Version), Times.Never); + _telemetryService.Verify(t => t.TrackPackageHasIncorrectHash(PackageIdentity.Id, PackageIdentity.Version), Times.Never); + } + } +} diff --git a/tests/NgTests/PackageMonitoringStatusAccessConditionHelperTests.cs b/tests/NgTests/PackageMonitoringStatusAccessConditionHelperTests.cs new file mode 100644 index 000000000..9044496eb --- /dev/null +++ b/tests/NgTests/PackageMonitoringStatusAccessConditionHelperTests.cs @@ -0,0 +1,142 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.WindowsAzure.Storage; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Services.Metadata.Catalog.Persistence; +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace NgTests +{ + public class PackageMonitoringStatusAccessConditionHelperTests + { + [Fact] + public void FromContentReturnsEmptyIfNoETag() + { + var content = new StringStorageContent("content"); + PackageMonitoringStatusTestUtility.AssertAccessCondition( + AccessCondition.GenerateEmptyCondition(), + PackageMonitoringStatusAccessConditionHelper.FromContent(content)); + } + + [Fact] + public void FromContentReturnsEmptyIfNullETag() + { + var content = new StringStorageContentWithETag("content", null); + PackageMonitoringStatusTestUtility.AssertAccessCondition( + AccessCondition.GenerateEmptyCondition(), + PackageMonitoringStatusAccessConditionHelper.FromContent(content)); + } + + [Fact] + public void FromContentReturnsMatchIfETag() + { + var eTag = "etag"; + var content = new StringStorageContentWithETag("content", eTag); + PackageMonitoringStatusTestUtility.AssertAccessCondition( + AccessCondition.GenerateIfMatchCondition(eTag), + PackageMonitoringStatusAccessConditionHelper.FromContent(content)); + } + + public static IEnumerable UpdateFromExistingUpdatesExistingStatus_Data + { + get + { + foreach (var previousState in Enum.GetValues(typeof(PackageState)).Cast()) + { + foreach (var accessCondition in + new[] + { + AccessCondition.GenerateIfNotExistsCondition(), + AccessCondition.GenerateIfMatchCondition("howdy"), + AccessCondition.GenerateEmptyCondition() + }) + { + foreach (var newState in Enum.GetValues(typeof(PackageState)).Cast()) + { + yield return new object[] { previousState, accessCondition, newState }; + } + } + } + } + } + + [Theory] + [MemberData(nameof(UpdateFromExistingUpdatesExistingStatus_Data))] + public void UpdateFromExistingUpdatesExistingStatus(PackageState previousState, AccessCondition accessCondition, PackageState newState) + { + // Arrange + var feedPackageIdentity = new FeedPackageIdentity("howdy", "3.4.6"); + + var existingStatus = PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + feedPackageIdentity.Id, + feedPackageIdentity.Version, + PackageMonitoringStatusTestUtility.GetTestResultFromPackageState(previousState)); + + existingStatus.AccessCondition = accessCondition; + + var newStatus = PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + feedPackageIdentity.Id, + feedPackageIdentity.Version, + PackageMonitoringStatusTestUtility.GetTestResultFromPackageState(newState)); + + // Act + PackageMonitoringStatusAccessConditionHelper.UpdateFromExisting(newStatus, existingStatus); + + // Assert + foreach (var state in Enum.GetValues(typeof(PackageState)).Cast()) + { + PackageMonitoringStatusTestUtility.AssertAccessCondition( + state == previousState ? accessCondition : AccessCondition.GenerateIfNotExistsCondition(), + newStatus.ExistingState[state]); + } + } + + public static IEnumerable UpdateFromExistingUpdatesExistingStatusWhenNull_Data + { + get + { + foreach (var newState in Enum.GetValues(typeof(PackageState)).Cast()) + { + yield return new object[] { newState }; + } + } + } + + [Theory] + [MemberData(nameof(UpdateFromExistingUpdatesExistingStatusWhenNull_Data))] + public void UpdateFromExistingUpdatesExistingStatusWhenNull(PackageState newState) + { + // Arrange + var feedPackageIdentity = new FeedPackageIdentity("howdy", "3.4.6"); + + var newStatus = PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + feedPackageIdentity.Id, + feedPackageIdentity.Version, + PackageMonitoringStatusTestUtility.GetTestResultFromPackageState(newState)); + + // Act + PackageMonitoringStatusAccessConditionHelper.UpdateFromExisting(newStatus, null); + + // Assert + foreach (var state in Enum.GetValues(typeof(PackageState)).Cast()) + { + PackageMonitoringStatusTestUtility.AssertAccessCondition( + AccessCondition.GenerateIfNotExistsCondition(), + newStatus.ExistingState[state]); + } + } + + [Fact] + public void FromUnknownReturnsEmptyCondition() + { + PackageMonitoringStatusTestUtility.AssertAccessCondition( + AccessCondition.GenerateEmptyCondition(), + PackageMonitoringStatusAccessConditionHelper.FromUnknown()); + } + } +} diff --git a/tests/NgTests/PackageMonitoringStatusServiceTests.cs b/tests/NgTests/PackageMonitoringStatusServiceTests.cs new file mode 100644 index 000000000..4344f964c --- /dev/null +++ b/tests/NgTests/PackageMonitoringStatusServiceTests.cs @@ -0,0 +1,447 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.WindowsAzure.Storage; +using Moq; +using Newtonsoft.Json; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Versioning; +using Xunit; + +namespace NgTests +{ + public class PackageMonitoringStatusServiceTests + { + public IPackageMonitoringStatusService Service { get; } + + public PackageMonitoringStatusServiceTests() + { + Service = new PackageMonitoringStatusService( + new MemoryStorageFactory(), + new Mock>().Object); + } + + private string GetPackageFileName(string packageId, string packageVersion) + { + return $"{packageId.ToLowerInvariant()}/{packageId.ToLowerInvariant()}.{packageVersion.ToLowerInvariant()}.json"; + } + + [Fact] + public async Task UpdateSavesNewStatus() + { + // Arrange + var feedPackageIdentity = new FeedPackageIdentity("howdy", "3.4.6"); + var packageValidationResult = new PackageValidationResult( + new PackageIdentity(feedPackageIdentity.Id, new NuGetVersion(feedPackageIdentity.Version)), + null, + null, + Enumerable.Empty()); + + var status = new PackageMonitoringStatus(packageValidationResult); + + var storageFactory = new MemoryStorageFactory(); + + var statusService = new PackageMonitoringStatusService( + storageFactory, + new Mock>().Object); + + // Act + await statusService.UpdateAsync(status, CancellationToken.None); + + // Assert + Assert.True( + storageFactory.Create( + PackageState.Valid.ToString().ToLowerInvariant()) + .Exists(GetPackageFileName(feedPackageIdentity.Id, feedPackageIdentity.Version))); + } + + public static IEnumerable UpdateDeletesOldStatuses_Data + { + get + { + yield return new object[] { null }; + foreach (var state in Enum.GetValues(typeof(PackageState)).Cast()) + { + yield return new object[] { state }; + } + } + } + + [Theory] + [MemberData(nameof(UpdateDeletesOldStatuses_Data))] + public async Task UpdateDeletesOldStatuses(PackageState? previousState) + { + // Arrange + var feedPackageIdentity = new FeedPackageIdentity("howdy", "3.4.6"); + + var packageValidationResult = new PackageValidationResult( + new PackageIdentity(feedPackageIdentity.Id, new NuGetVersion(feedPackageIdentity.Version)), + null, + null, + Enumerable.Empty()); + var status = new PackageMonitoringStatus(packageValidationResult); + + var storageFactory = new MemoryStorageFactory(); + + var statusService = new PackageMonitoringStatusService( + storageFactory, + new Mock>().Object); + + var etag = "theETag"; + foreach (var state in Enum.GetValues(typeof(PackageState)).Cast()) + { + if (previousState != state) + { + status.ExistingState[state] = AccessCondition.GenerateIfNotExistsCondition(); + continue; + } + + var content = new StringStorageContentWithETag("{}", etag); + await SaveToStorage(storageFactory, state, feedPackageIdentity, content); + status.ExistingState[state] = AccessCondition.GenerateIfMatchCondition(etag); + } + + // Act + await statusService.UpdateAsync(status, CancellationToken.None); + + // Assert + foreach (var state in Enum.GetValues(typeof(PackageState)).Cast()) + { + Assert.Equal( + state == status.State, + DoesPackageExists(storageFactory, state, feedPackageIdentity)); + } + + PackageMonitoringStatusTestUtility.AssertStatus( + status, + await statusService.GetAsync(feedPackageIdentity, CancellationToken.None)); + } + + [Fact] + public async Task GetByPackageNoResults() + { + // Arrange + var statusService = new PackageMonitoringStatusService( + new MemoryStorageFactory(), + new Mock>().Object); + + var desiredPackageId = "missingpackage"; + var desiredPackageVersion = "9.1.1"; + + // Act + var status = await statusService.GetAsync(new FeedPackageIdentity(desiredPackageId, desiredPackageVersion), CancellationToken.None); + + // Assert + Assert.Null(status); + } + + [Fact] + public async Task GetByPackageWithPackageValidationResult() + { + // Arrange + var statusService = new PackageMonitoringStatusService( + new MemoryStorageFactory(), + new Mock>().Object); + + var undesiredStatuses = new PackageMonitoringStatus[] + { + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "json.newtonsoft", + "1.0.9", + TestResult.Pass), + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "json.newtonsoft.json", + "1.0.9.1", + TestResult.Fail), + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "j.n.j", + "1.9.1", + TestResult.Skip), + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "newtonsoft.json", + "9.0.2", + TestResult.Pass) + }; + + var desiredPackageId = "newtonsoft.json"; + var desiredPackageVersion = "9.0.1"; + var desiredStatus = PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + desiredPackageId, + desiredPackageVersion, + TestResult.Pass); + + await statusService.UpdateAsync(desiredStatus, CancellationToken.None); + await Task.WhenAll(undesiredStatuses.Select(s => statusService.UpdateAsync(s, CancellationToken.None))); + + // Act + var status = await statusService.GetAsync(new FeedPackageIdentity(desiredPackageId, desiredPackageVersion), CancellationToken.None); + + // Assert + PackageMonitoringStatusTestUtility.AssertStatus(desiredStatus, status); + } + + [Fact] + public async Task GetByPackageWithException() + { + // Arrange + var statusService = new PackageMonitoringStatusService( + new MemoryStorageFactory(), + new Mock>().Object); + + var undesiredStatuses = new PackageMonitoringStatus[] + { + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "json.newtonsoft", + "1.0.9", + TestResult.Pass), + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "json.newtonsoft.json", + "1.0.9.1", + TestResult.Fail), + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "j.n.j", + "1.9.1", + TestResult.Skip), + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "newtonsoft.json", + "9.0.2", + TestResult.Pass) + }; + + var desiredPackageId = "newtonsoft.json"; + var desiredPackageVersion = "9.0.1"; + var desiredStatus = PackageMonitoringStatusTestUtility.CreateStatusWithException(desiredPackageId, desiredPackageVersion); + + await statusService.UpdateAsync(desiredStatus, CancellationToken.None); + await Task.WhenAll(undesiredStatuses.Select(s => statusService.UpdateAsync(s, CancellationToken.None))); + + // Act + var status = await statusService.GetAsync(new FeedPackageIdentity(desiredPackageId, desiredPackageVersion), CancellationToken.None); + + // Assert + PackageMonitoringStatusTestUtility.AssertStatus(desiredStatus, status); + } + + [Fact] + public async Task GetByPackageDeserializationException() + { + // Arrange + var desiredPackageId = "brokenpackage"; + var desiredPackageVersion = "99.9.99"; + + var storageFactory = new MemoryStorageFactory(); + var storage = storageFactory.Create(PackageState.Valid.ToString().ToLowerInvariant()); + + await storage.SaveAsync( + storage.ResolveUri(GetPackageFileName(desiredPackageId, desiredPackageVersion)), + new StringStorageContent("this isn't json"), + CancellationToken.None); + + var statusService = new PackageMonitoringStatusService( + storageFactory, + new Mock>().Object); + + // Act + var status = await statusService.GetAsync(new FeedPackageIdentity(desiredPackageId, desiredPackageVersion), CancellationToken.None); + + // Assert + Assert.Equal(desiredPackageId, status.Package.Id); + Assert.Equal(desiredPackageVersion, status.Package.Version); + Assert.IsType(status.ValidationException); + } + + public static IEnumerable GetByPackageDeletesOutdatedStatuses_Data + { + get + { + foreach (var latest in Enum.GetValues(typeof(PackageState)).Cast()) + { + foreach (var outdated in Enum.GetValues(typeof(PackageState)).Cast()) + { + yield return new object[] { latest, outdated }; + } + } + } + } + + [Theory] + [MemberData(nameof(GetByPackageDeletesOutdatedStatuses_Data))] + public async Task GetByPackageDeletesOutdatedStatuses(PackageState latest, PackageState outdated) + { + // Arrange + var storageFactory = new MemoryStorageFactory(); + var statusService = new PackageMonitoringStatusService( + storageFactory, + new Mock>().Object); + + var id = "howdyFriend"; + var version = "5.5.5"; + var package = new FeedPackageIdentity(id, version); + var outdatedStatus = PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + id, + version, + PackageMonitoringStatusTestUtility.GetTestResultFromPackageState(latest), + new DateTime(2019, 6, 10)); + + var latestStatus = PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + id, + version, + PackageMonitoringStatusTestUtility.GetTestResultFromPackageState(outdated), + new DateTime(2019, 6, 11)); + + await SaveToStorage(storageFactory, outdatedStatus); + await SaveToStorage(storageFactory, latestStatus); + + // Act + var status = await statusService.GetAsync(package, CancellationToken.None); + + // Assert + PackageMonitoringStatusTestUtility.AssertStatus(latestStatus, status); + Assert.Equal(latest == outdated, DoesPackageExists(storageFactory, outdatedStatus.State, package)); + Assert.True(DoesPackageExists(storageFactory, latestStatus.State, package)); + } + + [Fact] + public async Task GetByStateNoResults() + { + // Arrange + var statusService = new PackageMonitoringStatusService( + new MemoryStorageFactory(), + new Mock>().Object); + + // Act & Assert + foreach (var state in Enum.GetValues(typeof(PackageState)).Cast()) + { + var statuses = await statusService.GetAsync(state, CancellationToken.None); + Assert.Empty(statuses); + } + } + + [Fact] + public async Task GetByState() + { + // Arrange + var statusService = new PackageMonitoringStatusService( + new MemoryStorageFactory(), + new Mock>().Object); + + var expectedValidStatuses = new PackageMonitoringStatus[] + { + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "newtonsoft.json", "9.0.2", TestResult.Pass), + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "a.b", "1.2.3", TestResult.Pass), + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "newtonsoft.json", "6.0.8", TestResult.Skip), + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "a.b", "0.8.9", TestResult.Skip) + }; + + var expectedInvalidStatuses = new PackageMonitoringStatus[] + { + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "jQuery", + "3.1.2", + new ValidationResult[] + { + PackageMonitoringStatusTestUtility.CreateValidationResult( + TestResult.Fail, + new ValidationException("malarky!")) + }), + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "EntityFramework", + "6.1.2", + new ValidationResult[] + { + PackageMonitoringStatusTestUtility.CreateValidationResult( + TestResult.Fail, + new ValidationException("absurd!")) + }), + PackageMonitoringStatusTestUtility.CreateStatusWithException( + "NUnit", + "3.6.1") + }; + + var expectedUnknownStatuses = new PackageMonitoringStatus[] + { + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "xunit", "2.4.1", TestResult.Pending), + PackageMonitoringStatusTestUtility.CreateStatusWithPackageValidationResult( + "a.b", "99.9.99", TestResult.Pending) + }; + + foreach (var expectedValidStatus in expectedValidStatuses) + { + await statusService.UpdateAsync(expectedValidStatus, CancellationToken.None); + } + + foreach (var expectedInvalidStatus in expectedInvalidStatuses) + { + await statusService.UpdateAsync(expectedInvalidStatus, CancellationToken.None); + } + + foreach (var expectedSkippedStatus in expectedUnknownStatuses) + { + await statusService.UpdateAsync(expectedSkippedStatus, CancellationToken.None); + } + + // Act + var validStatuses = await statusService.GetAsync(PackageState.Valid, CancellationToken.None); + var invalidStatuses = await statusService.GetAsync(PackageState.Invalid, CancellationToken.None); + var unknownStatuses = await statusService.GetAsync(PackageState.Unknown, CancellationToken.None); + + // Assert + PackageMonitoringStatusTestUtility.AssertAll( + expectedValidStatuses.OrderBy(s => s.Package.Id).ThenBy(s => s.Package.Version), + validStatuses.OrderBy(s => s.Package.Id).ThenBy(s => s.Package.Version), + PackageMonitoringStatusTestUtility.AssertStatus); + + PackageMonitoringStatusTestUtility.AssertAll( + expectedInvalidStatuses.OrderBy(s => s.Package.Id).ThenBy(s => s.Package.Version), + invalidStatuses.OrderBy(s => s.Package.Id).ThenBy(s => s.Package.Version), + PackageMonitoringStatusTestUtility.AssertStatus); + + PackageMonitoringStatusTestUtility.AssertAll( + expectedUnknownStatuses.OrderBy(s => s.Package.Id).ThenBy(s => s.Package.Version), + unknownStatuses.OrderBy(s => s.Package.Id).ThenBy(s => s.Package.Version), + PackageMonitoringStatusTestUtility.AssertStatus); + } + + private Task SaveToStorage(MemoryStorageFactory storageFactory, PackageMonitoringStatus status) + { + var json = JsonConvert.SerializeObject(status, JsonSerializerUtility.SerializerSettings); + var content = new StringStorageContentWithAccessCondition( + json, + AccessCondition.GenerateEmptyCondition(), + "application/json"); + + return SaveToStorage(storageFactory, status.State, status.Package, content); + } + + private Task SaveToStorage(MemoryStorageFactory storageFactory, PackageState state, FeedPackageIdentity package, StorageContent content) + { + var stateName = Enum.GetName(typeof(PackageState), state); + var storage = storageFactory.Create(stateName.ToLowerInvariant()); + var packageFileName = GetPackageFileName(package.Id, package.Version); + return storage.SaveAsync(storage.ResolveUri(packageFileName), content, CancellationToken.None); + } + + private bool DoesPackageExists(MemoryStorageFactory storageFactory, PackageState state, FeedPackageIdentity package) + { + var stateName = Enum.GetName(typeof(PackageState), state); + var storage = storageFactory.Create(stateName.ToLowerInvariant()); + var packageFileName = GetPackageFileName(package.Id, package.Version); + return storage.Exists(packageFileName); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/PackageMonitoringStatusTestUtility.cs b/tests/NgTests/PackageMonitoringStatusTestUtility.cs new file mode 100644 index 000000000..0598a8b15 --- /dev/null +++ b/tests/NgTests/PackageMonitoringStatusTestUtility.cs @@ -0,0 +1,278 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.WindowsAzure.Storage; +using Newtonsoft.Json.Linq; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Versioning; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace NgTests +{ + public static class PackageMonitoringStatusTestUtility + { + public static ValidationResult CreateValidationResult(TestResult result, Exception e) + { + return new DummyValidator(result, e).Validate(); + } + + public static CatalogIndexEntry CreateCatalogIndexEntry(string id, string version, DateTime commitTimestamp) + { + return new CatalogIndexEntry( + new UriBuilder() { Path = $"{id.ToLowerInvariant()}/{id.ToLowerInvariant()}.{version.ToLowerInvariant()}" }.Uri, + CatalogConstants.NuGetPackageDetails, + Guid.NewGuid().ToString(), + commitTimestamp, + new PackageIdentity(id, new NuGetVersion(version))); + } + + public static DeletionAuditEntry CreateDeletionAuditEntry(string id, string version, DateTime commitTimestamp) + { + return new DeletionAuditEntry( + new UriBuilder() { Path = $"auditing/{id}/{version}/{Guid.NewGuid().ToString()}{DeletionAuditEntry.FileNameSuffixes[0]}" }.Uri, + JObject.Parse("{\"help\":\"i'm trapped in a json factory!\"}"), + id, + version, + commitTimestamp); + } + + public static PackageMonitoringStatus CreateStatusWithPackageValidationResult( + string packageId, + string packageVersion, + TestResult result, + DateTime? commitTimestamp = null) + { + return CreateStatusWithPackageValidationResult( + packageId, + packageVersion, + new[] { CreateValidationResult(result, null) }, + commitTimestamp); + } + + public static PackageMonitoringStatus CreateStatusWithPackageValidationResult( + string packageId, + string packageVersion, + IEnumerable results, + DateTime? commitTimestamp = null) + { + commitTimestamp = commitTimestamp ?? new DateTime(2019, 6, 10); + var version = new NuGetVersion(packageVersion); + + var aggregateValidationResult = new DummyAggregateValidator(results).Validate(); + + var packageValidationResult = new PackageValidationResult( + new PackageIdentity(packageId, version), + new CatalogIndexEntry[] { + CreateCatalogIndexEntry(packageId, packageVersion, commitTimestamp.Value), + CreateCatalogIndexEntry(packageId, packageVersion, commitTimestamp.Value), + CreateCatalogIndexEntry(packageId, packageVersion, commitTimestamp.Value) + }, + new DeletionAuditEntry[] { + CreateDeletionAuditEntry(packageId, packageVersion, commitTimestamp.Value), + CreateDeletionAuditEntry(packageId, packageVersion, commitTimestamp.Value), + CreateDeletionAuditEntry(packageId, packageVersion, commitTimestamp.Value) + }, + new AggregateValidationResult[] { aggregateValidationResult }); + + return new PackageMonitoringStatus(packageValidationResult); + } + + public static PackageMonitoringStatus CreateStatusWithException(string packageId, string packageVersion) + { + return new PackageMonitoringStatus(new FeedPackageIdentity(packageId, packageVersion), new Exception()); + } + + public static TestResult GetTestResultFromPackageState(PackageState state) + { + switch (state) + { + case PackageState.Invalid: + return TestResult.Fail; + case PackageState.Valid: + return TestResult.Pass; + case PackageState.Unknown: + return TestResult.Pending; + default: + throw new ArgumentException(nameof(state)); + } + } + + public static void AssertFieldEqual( + TParent expected, + TParent actual, + Func accessor) + { + Assert.Equal(accessor(expected), accessor(actual)); + } + + public static void AssertFieldEqual( + TParent expected, + TParent actual, + Func accessor, + Action assert) + { + assert(accessor(expected), accessor(actual)); + } + + public static void AssertFieldEqual( + TParent expected, + TParent actual, + Func> accessor, + Action assert) + { + AssertAll(accessor(expected), accessor(actual), assert); + } + + public static void AssertStatus(PackageMonitoringStatus expected, PackageMonitoringStatus actual) + { + AssertFieldEqual(expected, actual, i => i.Package.Id); + AssertFieldEqual(expected, actual, i => i.Package.Version); + AssertFieldEqual(expected, actual, i => i.State); + + AssertFieldEqual(expected, actual, i => i.ValidationResult, AssertPackageValidationResult); + AssertFieldEqual(expected, actual, i => i.ValidationException, AssertException); + } + + public static void AssertException(Exception expected, Exception actual) + { + if (expected == null) + { + Assert.Null(actual); + + return; + } + + AssertFieldEqual(expected, actual, i => i.Message); + AssertFieldEqual(expected, actual, i => i.StackTrace); + AssertFieldEqual(expected, actual, i => i.Data, AssertDictionary); + AssertFieldEqual(expected, actual, i => i.InnerException, AssertException); + } + + public static void AssertDictionary(IDictionary expected, IDictionary actual) + { + foreach (var expectedKey in expected.Keys) + { + Assert.True(actual.Contains(expectedKey)); + Assert.Equal(expected[expectedKey], actual[expectedKey]); + } + } + + public static void AssertPackageValidationResult(PackageValidationResult expected, PackageValidationResult actual) + { + if (expected == null) + { + Assert.Null(actual); + + return; + } + + AssertFieldEqual(expected, actual, i => i.Package.Id); + AssertFieldEqual(expected, actual, i => i.Package.Version); + + AssertFieldEqual(expected, actual, i => i.CatalogEntries, AssertCatalogIndexEntry); + AssertFieldEqual(expected, actual, i => i.DeletionAuditEntries, AssertDeletionAuditEntry); + + AssertFieldEqual(expected, actual, i => i.AggregateValidationResults, AssertAggregateValidationResult); + } + + public static void AssertCatalogIndexEntry(CatalogIndexEntry expected, CatalogIndexEntry actual) + { + if (expected == null) + { + Assert.Null(actual); + + return; + } + + AssertFieldEqual(expected, actual, i => i.Uri); + AssertFieldEqual(expected, actual, i => i.Types); + AssertFieldEqual(expected, actual, i => i.Id); + AssertFieldEqual(expected, actual, i => i.Version); + AssertFieldEqual(expected, actual, i => i.CommitId); + AssertFieldEqual(expected, actual, i => i.CommitTimeStamp); + } + + public static void AssertDeletionAuditEntry(DeletionAuditEntry expected, DeletionAuditEntry actual) + { + if (expected == null) + { + Assert.Null(actual); + + return; + } + + AssertFieldEqual(expected, actual, i => i.PackageId); + AssertFieldEqual(expected, actual, i => i.PackageVersion); + AssertFieldEqual(expected, actual, i => i.Record); + AssertFieldEqual(expected, actual, i => i.TimestampUtc); + AssertFieldEqual(expected, actual, i => i.Uri); + } + + public static void AssertAggregateValidationResult(AggregateValidationResult expected, AggregateValidationResult actual) + { + if (expected == null) + { + Assert.Null(actual); + + return; + } + + AssertFieldEqual(expected, actual, i => i.AggregateValidator.Name); + AssertFieldEqual(expected, actual, i => i.ValidationResults, AssertValidationResult); + } + + public static void AssertValidationResult(ValidationResult expected, ValidationResult actual) + { + if (expected == null) + { + Assert.Null(actual); + + return; + } + + AssertFieldEqual(expected, actual, i => i.Validator.Name); + AssertFieldEqual(expected, actual, i => i.Result); + + AssertFieldEqual(expected, actual, i => i.Exception, AssertException); + } + + public static void AssertAccessCondition(AccessCondition expected, AccessCondition actual) + { + AssertFieldEqual(expected, actual, i => i.LeaseId); + AssertFieldEqual(expected, actual, i => i.IfModifiedSinceTime); + AssertFieldEqual(expected, actual, i => i.IfNoneMatchETag); + AssertFieldEqual(expected, actual, i => i.IfMatchETag); + AssertFieldEqual(expected, actual, i => i.IfAppendPositionEqual); + AssertFieldEqual(expected, actual, i => i.IfSequenceNumberEqual); + AssertFieldEqual(expected, actual, i => i.IfSequenceNumberLessThanOrEqual); + AssertFieldEqual(expected, actual, i => i.IfSequenceNumberLessThan); + AssertFieldEqual(expected, actual, i => i.IfMaxSizeLessThanOrEqual); + AssertFieldEqual(expected, actual, i => i.IfNotModifiedSinceTime); + } + + public static void AssertAll(IEnumerable expecteds, IEnumerable actuals, Action assert) + { + if (expecteds == null) + { + Assert.Null(actuals); + + return; + } + + Assert.Equal(expecteds.Count(), actuals.Count()); + var expectedsArray = expecteds.ToArray(); + var actualsArray = actuals.ToArray(); + for (int i = 0; i < expecteds.Count(); i++) + { + assert(expectedsArray[i], actualsArray[i]); + } + } + } +} diff --git a/tests/NgTests/PackageMonitoringStatusTests.cs b/tests/NgTests/PackageMonitoringStatusTests.cs new file mode 100644 index 000000000..c6380a523 --- /dev/null +++ b/tests/NgTests/PackageMonitoringStatusTests.cs @@ -0,0 +1,117 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Versioning; +using Xunit; + +namespace NgTests +{ + public class PackageMonitoringStatusTests + { + public class WithPackageValidationResult + { + [Fact] + public void ThrowsIfNullArgument() + { + Assert.Throws(() => new PackageMonitoringStatus(null)); + } + + [Fact] + public void Valid() + { + var validationResults = new ValidationResult[] + { + new ValidationResult(null, TestResult.Pass), + new ValidationResult(null, TestResult.Skip) + }; + + var aggregateValidationResults = new AggregateValidationResult( + null, + validationResults); + + var packageValidationResult = new PackageValidationResult( + new PackageIdentity("testPackage", new NuGetVersion(4, 5, 6)), + null, + null, + new AggregateValidationResult[] { aggregateValidationResults }); + + var status = new PackageMonitoringStatus(packageValidationResult); + + Assert.Equal(PackageState.Valid, status.State); + } + + [Fact] + public void Invalid() + { + var validationResults = new ValidationResult[] + { + new ValidationResult(null, TestResult.Fail) + }; + + var aggregateValidationResults = new AggregateValidationResult( + null, + validationResults); + + var packageValidationResult = new PackageValidationResult( + new PackageIdentity("testPackage", new NuGetVersion(4, 5, 6)), + null, + null, + new AggregateValidationResult[] { aggregateValidationResults }); + + var status = new PackageMonitoringStatus(packageValidationResult); + + Assert.Equal(PackageState.Invalid, status.State); + } + + [Fact] + public void Unknown() + { + var validationResults = new ValidationResult[] + { + new ValidationResult(null, TestResult.Pass), + new ValidationResult(null, TestResult.Pending) + }; + + var aggregateValidationResults = new AggregateValidationResult( + null, + validationResults); + + var packageValidationResult = new PackageValidationResult( + new PackageIdentity("testPackage", new NuGetVersion(4, 5, 6)), + null, + null, + new AggregateValidationResult[] { aggregateValidationResults }); + + var status = new PackageMonitoringStatus(packageValidationResult); + + Assert.Equal(PackageState.Unknown, status.State); + } + } + + public class WithException + { + [Fact] + public void ThrowsIfNullArgument() + { + Assert.Throws( + () => new PackageMonitoringStatus(new FeedPackageIdentity("hi", "1.0.0"), null)); + Assert.Throws( + () => new PackageMonitoringStatus(null, new Exception())); + Assert.Throws( + () => new PackageMonitoringStatus(null, null)); + } + + [Fact] + public void Invalid() + { + var status = new PackageMonitoringStatus(new FeedPackageIdentity("hello", "2.1.0"), new Exception()); + + Assert.Equal(PackageState.Invalid, status.State); + } + } + } +} diff --git a/tests/NgTests/PackageTimestampMetadataTests.cs b/tests/NgTests/PackageTimestampMetadataTests.cs new file mode 100644 index 000000000..9de23bcdf --- /dev/null +++ b/tests/NgTests/PackageTimestampMetadataTests.cs @@ -0,0 +1,185 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using NgTests.Data; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Versioning; +using Xunit; + +namespace NgTests.Validation +{ + public class PackageTimestampMetadataTests + { + public static IEnumerable Last_PicksLatest_data + { + get + { + yield return new object[] + { + true, + DateTime.MinValue, + DateTime.MaxValue, + null + }; + + yield return new object[] + { + true, + DateTime.MaxValue, + DateTime.MinValue, + null + }; + + yield return new object[] + { + false, + null, + null, + DateTime.MinValue + }; + + yield return new object[] + { + false, + null, + null, + null + }; + } + } + + [Theory] + [MemberData(nameof(Last_PicksLatest_data))] + public void Last_PicksLatest(bool exists, DateTime? created, DateTime? lastEdited, DateTime? deleted) + { + // Act + PackageTimestampMetadata package; + + if (exists) + { + package = PackageTimestampMetadata.CreateForExistingPackage(created.Value, lastEdited.Value); + } + else + { + package = PackageTimestampMetadata.CreateForMissingPackage(deleted); + } + + // Assert + Assert.True(created == null || package.Last >= created); + Assert.True(lastEdited == null || package.Last >= lastEdited); + Assert.True(deleted == null || package.Last >= deleted); + } + + [Fact] + public async Task FromCatalogEntry_HandlesCreatedLastEdited() + { + // Arrange + var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); + var client = await CreateDummyClient(catalogStorage); + + var expectedTimestamp = DateTime.Parse("2015-01-01T00:00:00"); + var commitTimeStamp1 = DateTime.ParseExact( + "2015-10-12T10:08:54.1506742Z", + CatalogConstants.CommitTimeStampFormat, + DateTimeFormatInfo.CurrentInfo, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal); + var commitTimeStamp2 = DateTime.ParseExact( + "2015-10-12T10:08:55.3335317Z", + CatalogConstants.CommitTimeStampFormat, + DateTimeFormatInfo.CurrentInfo, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal); + + var tasks = new Task[] + { + PackageTimestampMetadata.FromCatalogEntry( + client, + new CatalogIndexEntry( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/listedpackage.1.0.0.json"), + CatalogConstants.NuGetPackageDetails, + "9a37734f-1960-4c07-8934-c8bc797e35c1", + commitTimeStamp1, + new PackageIdentity("ListedPackage", new NuGetVersion("1.0.0")))), + PackageTimestampMetadata.FromCatalogEntry( + client, + new CatalogIndexEntry( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json"), + CatalogConstants.NuGetPackageDetails, + "9a37734f-1960-4c07-8934-c8bc797e35c1", + commitTimeStamp1, + new PackageIdentity("UnlistedPackage", new NuGetVersion("1.0.0")))), + PackageTimestampMetadata.FromCatalogEntry( + client, + new CatalogIndexEntry( + new Uri(catalogStorage.BaseAddress, "data/2015.10.12.10.08.55/listedpackage.1.0.1.json"), + CatalogConstants.NuGetPackageDetails, + "8a9e7694-73d4-4775-9b7a-20aa59b9773e", + commitTimeStamp2, + new PackageIdentity("ListedPackage", new NuGetVersion("1.0.1")))) + }; + + // Act + var entries = await Task.WhenAll(tasks); + + // Assert + foreach (var entry in entries) + { + Assert.True(entry.Exists); + Assert.Equal(expectedTimestamp.Ticks, entry.Created.Value.Ticks); + Assert.Equal(expectedTimestamp.Ticks, entry.LastEdited.Value.Ticks); + Assert.Null(entry.Deleted); + Assert.Equal(expectedTimestamp.Ticks, entry.Last.Value.Ticks); + } + } + + [Fact] + public async Task FromCatalogEntry_HandlesDeleted_NotNull() + { + // Arrange + var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); + var client = await CreateDummyClient(catalogStorage); + + var expectedTimestamp = DateTime.Parse("2015-01-01T01:01:01.0748028"); + + var uri = new Uri(catalogStorage.BaseAddress, "data/2015.10.13.06.40.07/otherpackage.1.0.0.json"); + var catalogIndexEntry = new CatalogIndexEntry( + uri, + CatalogConstants.NuGetPackageDelete, + "afc8c1f4-486e-4142-b3ec-cf5841eb8883", + DateTime.ParseExact( + "2015-10-13T06:40:07.7850657Z", + CatalogConstants.CommitTimeStampFormat, + DateTimeFormatInfo.CurrentInfo, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal), + new PackageIdentity("OtherPackage", new NuGetVersion("1.0.0"))); + + // Act + var entry = await PackageTimestampMetadata.FromCatalogEntry(client, catalogIndexEntry); + + // Assert + Assert.False(entry.Exists); + Assert.Null(entry.Created); + Assert.Null(entry.LastEdited); + Assert.Equal(expectedTimestamp.Ticks, entry.Deleted.Value.Ticks); + Assert.Equal(expectedTimestamp.Ticks, entry.Last.Value.Ticks); + } + + private async Task CreateDummyClient(MemoryStorage catalogStorage) + { + var mockServer = new MockServerHttpClientHandler(); + + mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK))); + await mockServer.AddStorageAsync(catalogStorage); + + return new CollectorHttpClient(mockServer); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Packages/ListedPackage.1.0.0.zip b/tests/NgTests/Packages/ListedPackage.1.0.0.zip new file mode 100644 index 000000000..cdee2de99 Binary files /dev/null and b/tests/NgTests/Packages/ListedPackage.1.0.0.zip differ diff --git a/tests/NgTests/Packages/ListedPackage.1.0.1.zip b/tests/NgTests/Packages/ListedPackage.1.0.1.zip new file mode 100644 index 000000000..90e6c18b4 Binary files /dev/null and b/tests/NgTests/Packages/ListedPackage.1.0.1.zip differ diff --git a/tests/NgTests/Packages/OtherPackage.1.0.0.zip b/tests/NgTests/Packages/OtherPackage.1.0.0.zip new file mode 100644 index 000000000..2e26c2e12 Binary files /dev/null and b/tests/NgTests/Packages/OtherPackage.1.0.0.zip differ diff --git a/tests/NgTests/Packages/TestAuthorAndRepoSigned.leaf-1.1.0.0.nupkg b/tests/NgTests/Packages/TestAuthorAndRepoSigned.leaf-1.1.0.0.nupkg new file mode 100644 index 000000000..b26ac242a Binary files /dev/null and b/tests/NgTests/Packages/TestAuthorAndRepoSigned.leaf-1.1.0.0.nupkg differ diff --git a/tests/NgTests/Packages/TestRepoSigned.leaf-1.1.0.0.nupkg b/tests/NgTests/Packages/TestRepoSigned.leaf-1.1.0.0.nupkg new file mode 100644 index 000000000..a42a2a34f Binary files /dev/null and b/tests/NgTests/Packages/TestRepoSigned.leaf-1.1.0.0.nupkg differ diff --git a/tests/NgTests/Packages/TestSigned.leaf-1.1.0.0.nupkg b/tests/NgTests/Packages/TestSigned.leaf-1.1.0.0.nupkg new file mode 100644 index 000000000..173152f94 Binary files /dev/null and b/tests/NgTests/Packages/TestSigned.leaf-1.1.0.0.nupkg differ diff --git a/tests/NgTests/Packages/TestUnsigned.1.0.0.nupkg b/tests/NgTests/Packages/TestUnsigned.1.0.0.nupkg new file mode 100644 index 000000000..4ca0214c8 Binary files /dev/null and b/tests/NgTests/Packages/TestUnsigned.1.0.0.nupkg differ diff --git a/tests/NgTests/Packages/UnlistedPackage.1.0.0.zip b/tests/NgTests/Packages/UnlistedPackage.1.0.0.zip new file mode 100644 index 000000000..d811c4196 Binary files /dev/null and b/tests/NgTests/Packages/UnlistedPackage.1.0.0.zip differ diff --git a/tests/NgTests/Properties/AssemblyInfo.cs b/tests/NgTests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..946664d67 --- /dev/null +++ b/tests/NgTests/Properties/AssemblyInfo.cs @@ -0,0 +1,17 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NgTests")] +[assembly: AssemblyDescription("NgTests")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany(".NET Foundation")] +[assembly: AssemblyProduct("NuGet")] +[assembly: AssemblyCopyright("\x00a9 .NET Foundation. All rights reserved.")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] +[assembly: ComVisible(false)] +[assembly: Guid("05c1c78a-9966-4922-9065-a099023e7366")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] \ No newline at end of file diff --git a/tests/NgTests/SortingIdVersionCollectorTests.cs b/tests/NgTests/SortingIdVersionCollectorTests.cs new file mode 100644 index 000000000..71f9ac00f --- /dev/null +++ b/tests/NgTests/SortingIdVersionCollectorTests.cs @@ -0,0 +1,137 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Moq; +using Newtonsoft.Json.Linq; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Versioning; +using Xunit; + +namespace NgTests +{ + public class SortingIdVersionCollectorTests + { + public static IEnumerable BatchesData + { + get + { + // Different id, same version + yield return new object[] + { + new List + { + CreatePackage("test1", "1.0.0"), + CreatePackage("test2", "1.0.0"), + CreatePackage("test3", "2.0.0"), + CreatePackage("test4", "2.0.0") + } + }; + + // Same id, different version + yield return new object[] + { + new List + { + CreatePackage("test1", "1.0.0"), + CreatePackage("test1", "2.0.0"), + CreatePackage("test1", "3.0.0"), + CreatePackage("test1", "4.0.0") + } + }; + + // Same id, same version + yield return new object[] + { + new List + { + CreatePackage("test1", "1.0.0"), + CreatePackage("test1", "1.0.0"), + CreatePackage("test2", "2.0.0"), + CreatePackage("test2", "2.0.0") + } + }; + } + } + + [Theory] + [MemberData(nameof(BatchesData))] + public async Task OnProcessBatch_BatchesCorrectly(IEnumerable items) + { + // Arrange + var collectorMock = new Mock() + { + CallBase = true + }; + + var seenPackages = new List(); + + collectorMock + .Setup(x => x.OverridableProcessSortedBatch(It.IsAny>>())) + .Returns>>( + (pair) => + { + // Assert + Assert.DoesNotContain( + seenPackages, + (p) => + { + return p.Id == pair.Key.Id && p.Version == pair.Key.Version; + }); + + seenPackages.Add(new FeedPackageIdentity(pair.Key.Id, pair.Key.Version)); + + return Task.FromResult(0); + }); + + // Act + var result = await collectorMock.Object.OnProcessBatchAsync(items); + } + + private static CatalogCommitItem CreatePackage(string id, string version) + { + var context = TestUtility.CreateCatalogContextJObject(); + var packageIdentity = new PackageIdentity(id, new NuGetVersion(version)); + var commitItem = TestUtility.CreateCatalogCommitItemJObject(DateTime.UtcNow, packageIdentity); + + return CatalogCommitItem.Create(context, commitItem); + } + + public class TestableSortingIdVersionCollector : SortingIdVersionCollector + { + public TestableSortingIdVersionCollector() + : base( + new Uri("https://nuget.test"), + Mock.Of(), + handlerFunc: null) + { + } + + public Task OnProcessBatchAsync(IEnumerable items) + { + return base.OnProcessBatchAsync(null, items, null, DateTime.MinValue, false, CancellationToken.None); + } + + protected override Task ProcessSortedBatchAsync( + CollectorHttpClient client, + KeyValuePair> sortedBatch, + JToken context, + CancellationToken cancellationToken) + { + return OverridableProcessSortedBatch(sortedBatch); + } + + public virtual Task OverridableProcessSortedBatch( + KeyValuePair> sortedBatch) + { + return Task.FromResult(0); + } + } + } +} \ No newline at end of file diff --git a/tests/NgTests/StorageFactoryTests.cs b/tests/NgTests/StorageFactoryTests.cs new file mode 100644 index 000000000..180bbc811 --- /dev/null +++ b/tests/NgTests/StorageFactoryTests.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.ComponentModel; +using Ng; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit; + +namespace NgTests +{ + public class StorageFactoryTests + { + [Theory] + [Description("The regular azure factory should not compress the content if.")] + [InlineData("http://localhost/reg", "testAccount", "DummyDUMMYpZxLeDumMyyN52gJj+ZlGE0ipRi9PaTcn9AU4epwvsngE5rLSMk9TwpazxUtzeyBnFeWFAdummyw==", "testContainer", "testStoragePath", "azure")] + public void AzureFactory(string storageBaseAddress, + string storageAccountName, + string storageKeyValue, + string storageContainer, + string storagePath, + string storageType) + { + Dictionary arguments = new Dictionary() + { + { Arguments.StorageBaseAddress, storageBaseAddress }, + { Arguments.StorageAccountName, storageAccountName }, + { Arguments.StorageKeyValue, storageKeyValue }, + { Arguments.StorageContainer, storageContainer }, + { Arguments.StoragePath, storagePath }, + { Arguments.StorageType, storageType} + }; + + StorageFactory factory = CommandHelpers.CreateStorageFactory(arguments, true); + AzureStorageFactory azureFactory = factory as AzureStorageFactory; + // Assert + Assert.True(azureFactory != null, "The CreateCompressedStorageFactory should return an AzureStorageFactory type."); + Assert.False(azureFactory.CompressContent, "The azure storage factory should not compress the content."); + } + } +} diff --git a/tests/NgTests/TestableDb2CatalogJob.cs b/tests/NgTests/TestableDb2CatalogJob.cs new file mode 100644 index 000000000..42da0a428 --- /dev/null +++ b/tests/NgTests/TestableDb2CatalogJob.cs @@ -0,0 +1,63 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Moq; +using Ng.Jobs; +using NgTests.Infrastructure; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Persistence; +using Xunit.Abstractions; + +namespace NgTests +{ + public class TestableDb2CatalogJob + : Db2CatalogJob + { + private readonly HttpMessageHandler _handler; + + public TestableDb2CatalogJob( + HttpMessageHandler handler, + IStorage catalogStorage, + IStorage auditingStorage, + bool skipCreatedPackagesProcessing, + DateTime? startDate, + TimeSpan timeout, + int top, + bool verbose, + Mock galleryDatabaseMock, + PackageContentUriBuilder packageContentUriBuilder, + ITestOutputHelper testOutputHelper) + : base(new TestLoggerFactory(testOutputHelper), new Mock().Object, new Dictionary()) + { + _handler = handler; + + CatalogStorage = catalogStorage; + AuditingStorage = auditingStorage; + SkipCreatedPackagesProcessing = skipCreatedPackagesProcessing; + StartDate = startDate; + Timeout = timeout; + Top = top; + Verbose = verbose; + Destination = new Uri("https://nuget.test"); + + PackageContentUriBuilder = packageContentUriBuilder ?? throw new ArgumentNullException(nameof(galleryDatabaseMock)); + GalleryDatabaseQueryService = galleryDatabaseMock?.Object ?? throw new ArgumentNullException(nameof(galleryDatabaseMock)); + } + + protected override HttpClient CreateHttpClient() + { + return new HttpClient(_handler); + } + + public async Task RunOnceAsync(CancellationToken cancellationToken) + { + await RunInternalAsync(cancellationToken); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/UriUtilsTests.cs b/tests/NgTests/UriUtilsTests.cs new file mode 100644 index 000000000..910be5749 --- /dev/null +++ b/tests/NgTests/UriUtilsTests.cs @@ -0,0 +1,126 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Metadata.Catalog.Helpers; +using System; +using Xunit; +using NuGet.Versioning; + +namespace NgTests +{ + public class UriUtilsTests + { + [Theory] + // Packages + [InlineData("https://api.nuget.org/packages/newtonsoft.json.9.0.1.nupkg")] + [InlineData("https://api.nuget.org/packages/findpackagesbyid.1.0.0-findpackagesbyid.nupkg")] + [InlineData("https://api.nuget.org/packages/search.1.0.0-search.nupkg")] + [InlineData("https://api.nuget.org/packages/packages.1.0.0-packages.nupkg")] + // Index + [InlineData("https://api.nuget.org/v3/index.json")] + // Catalog + [InlineData("https://api.nuget.org/v3/catalog0/index.json")] + [InlineData("https://api.nuget.org/v3/catalog0/page0.json")] + [InlineData("https://api.nuget.org/v3/catalog0/data/2015.02.01.06.22.45/adam.jsgenerator.1.1.0.json")] + // Registration + [InlineData("https://api.nuget.org/v3/registration0/newtonsoft.json/index.json")] + [InlineData("https://api.nuget.org/v3/registration0/newtonsoft.json/9.0.1.json")] + [InlineData("https://api.nuget.org/v3/registration0/findpackagesbyid/1.0.0-findpackagesbyid.json")] + [InlineData("https://api.nuget.org/v3/registration0/search/1.0.0-search.json")] + [InlineData("https://api.nuget.org/v3/registration0/packages/1.0.0-packages.json")] + // Flat-Container + [InlineData("https://api.nuget.org/v3/flatcontainer/newtonsoft.json/index.json")] + [InlineData("https://api.nuget.org/v3/flatcontainer/newtonsoft.json/9.0.1/newtonsoft.json.9.0.1.nupkg")] + [InlineData("https://api.nuget.org/v3/flatcontainer/findpackagesbyid/1.0.0-findpackagesbyid/findpackagesbyid.1.0.0-findpackagesbyid.nupkg")] + [InlineData("https://api.nuget.org/v3/flatcontainer/search/1.0.0-search/search.1.0.0-search.nupkg")] + [InlineData("https://api.nuget.org/v3/flatcontainer/packages/1.0.0-packages/packages.1.0.0-packages.nupkg")] + // Search service + [InlineData("http://nuget-prod-0-v2v3search.cloudapp.net/search/query?q=id:'Newtonsoft.Json'")] + [InlineData("http://nuget-prod-0-v2v3search.cloudapp.net/query?q=id:'Newtonsoft.Json'")] + [InlineData("http://nuget-prod-0-v2v3search.cloudapp.net/search/query?q=id:'FindPackagesById'")] + [InlineData("http://nuget-prod-0-v2v3search.cloudapp.net/query?q=id:'FindPackagesById'")] + [InlineData("http://nuget-prod-0-v2v3search.cloudapp.net/search/query?q=id:'Search'")] + [InlineData("http://nuget-prod-0-v2v3search.cloudapp.net/query?q=id:'Search'")] + [InlineData("http://nuget-prod-0-v2v3search.cloudapp.net/search/query?q=id:'Packages'")] + [InlineData("http://nuget-prod-0-v2v3search.cloudapp.net/query?q=id:'Packages'")] + public void GetNonhijackableUri_ReturnsOriginalUri(string originalUriString) + { + // Arrange + var originalUri = new Uri(originalUriString); + + // Act + var newUri = UriUtils.GetNonhijackableUri(originalUri); + + // Assert + Assert.Equal(originalUriString, newUri.ToString()); + Assert.Same(originalUri, newUri); + } + + [Theory] + // Already has orderby + [InlineData("https://www.nuget.org/api/v2/Packages?$orderby=Id", "https://www.nuget.org/api/v2/Packages?$orderby=Version")] + [InlineData("https://www.nuget.org/api/v2/Packages?$orderby=Id&$filter=IsLatestVersion", "https://www.nuget.org/api/v2/Packages?$orderby=Version&$filter=IsLatestVersion")] + [InlineData("https://www.nuget.org/api/v2/Search()?$filter=IsAbsoluteLatestVersion&$skip=0&$top=30&searchTerm='pickles'&targetFramework='net45'&includePrerelease=true&$orderby=Id", "https://www.nuget.org/api/v2/Search()?$filter=IsAbsoluteLatestVersion&$skip=0&$top=30&searchTerm='pickles'&targetFramework='net45'&includePrerelease=true&$orderby=Version")] + [InlineData("https://www.nuget.org/api/v2/Search()?$filter=IsAbsoluteLatestVersion&$skip=0&$orderby=Id&$top=30&searchTerm='pickles'&targetFramework='net45'&includePrerelease=true", "https://www.nuget.org/api/v2/Search()?$filter=IsAbsoluteLatestVersion&$skip=0&$orderby=Version&$top=30&searchTerm='pickles'&targetFramework='net45'&includePrerelease=true")] + [InlineData("https://www.nuget.org/api/v2/FindPackagesById()?$filter=IsLatestVersion&$top=1&id='MySql.Data'&$orderby=Id", "https://www.nuget.org/api/v2/FindPackagesById()?$filter=IsLatestVersion&$top=1&id='MySql.Data'&$orderby=Version")] + [InlineData("https://www.nuget.org/api/v2/FindPackagesById()?$filter=IsLatestVersion&$orderby=Id&$top=1&id='MySql.Data'", "https://www.nuget.org/api/v2/FindPackagesById()?$filter=IsLatestVersion&$orderby=Version&$top=1&id='MySql.Data'")] + // Packages(Id='...',Version='...') + [InlineData("https://www.nuget.org/api/v2/Packages(Id='Microsoft.Owin.Security.Facebook',Version='3.0.1')", "https://www.nuget.org/api/v2/Packages?$filter=true and Id eq 'Microsoft.Owin.Security.Facebook' and NormalizedVersion eq '3.0.1'&semVerLevel=2.0.0")] + // Packages endpoint without orderby + [InlineData("https://www.nuget.org/api/v2/Packages", "https://www.nuget.org/api/v2/Packages?$orderby=Version")] + [InlineData("https://www.nuget.org/api/v2/Packages?$top=10", "https://www.nuget.org/api/v2/Packages?$top=10&$orderby=Version")] + // Search endpoint without orderby + [InlineData("https://www.nuget.org/api/v2/Search()", "https://www.nuget.org/api/v2/Search()?$orderby=Version")] + [InlineData("https://www.nuget.org/api/v2/Search()?$top=10", "https://www.nuget.org/api/v2/Search()?$top=10&$orderby=Version")] + // FindPackagesById endpoint without orderby + [InlineData("https://www.nuget.org/api/v2/FindPackagesById()", "https://www.nuget.org/api/v2/FindPackagesById()?$orderby=Version")] + [InlineData("https://www.nuget.org/api/v2/FindPackagesById()?id='Microsoft.Rest.ClientRuntime'", "https://www.nuget.org/api/v2/FindPackagesById()?id='Microsoft.Rest.ClientRuntime'&$orderby=Version")] + // Id and version contain names of other endpoints + [InlineData("https://www.nuget.org/api/v2/Packages(Id='FindPackagesById',Version='1.0.0')", "https://www.nuget.org/api/v2/Packages?$filter=true and Id eq 'FindPackagesById' and NormalizedVersion eq '1.0.0'&semVerLevel=2.0.0")] + [InlineData("https://www.nuget.org/api/v2/Packages(Id='Search',Version='1.0.0')", "https://www.nuget.org/api/v2/Packages?$filter=true and Id eq 'Search' and NormalizedVersion eq '1.0.0'&semVerLevel=2.0.0")] + [InlineData("https://www.nuget.org/api/v2/Packages(Id='Packages',Version='1.0.0')", "https://www.nuget.org/api/v2/Packages?$filter=true and Id eq 'Packages' and NormalizedVersion eq '1.0.0'&semVerLevel=2.0.0")] + [InlineData("https://www.nuget.org/api/v2/Packages(Id='abcd',Version='1.0.0-FindPackagesById')", "https://www.nuget.org/api/v2/Packages?$filter=true and Id eq 'abcd' and NormalizedVersion eq '1.0.0-FindPackagesById'&semVerLevel=2.0.0")] + [InlineData("https://www.nuget.org/api/v2/Packages(Id='abcd',Version='1.0.0-Search')", "https://www.nuget.org/api/v2/Packages?$filter=true and Id eq 'abcd' and NormalizedVersion eq '1.0.0-Search'&semVerLevel=2.0.0")] + [InlineData("https://www.nuget.org/api/v2/Packages(Id='abcd',Version='1.0.0-Packages')", "https://www.nuget.org/api/v2/Packages?$filter=true and Id eq 'abcd' and NormalizedVersion eq '1.0.0-Packages'&semVerLevel=2.0.0")] + [InlineData("https://www.nuget.org/api/v2/FindPackagesById()?id='Search'", "https://www.nuget.org/api/v2/FindPackagesById()?id='Search'&$orderby=Version")] + [InlineData("https://www.nuget.org/api/v2/FindPackagesById()?id='FindPackagesById'", "https://www.nuget.org/api/v2/FindPackagesById()?id='FindPackagesById'&$orderby=Version")] + [InlineData("https://www.nuget.org/api/v2/FindPackagesById()?id='Packages'", "https://www.nuget.org/api/v2/FindPackagesById()?id='Packages'&$orderby=Version")] + public void GetNonhijackableUri_ReturnsNonhijackableUri(string originalUriString, string expectedUriString) + { + // Arrange + var originalUri = new Uri(originalUriString); + + // Act + var newUri = UriUtils.GetNonhijackableUri(originalUri); + + // Assert + Assert.NotEqual(originalUriString, newUri.ToString()); + Assert.NotSame(originalUri, newUri); + + Assert.Equal(expectedUriString, newUri.ToString()); + } + + [Theory] + [InlineData("1.00")] + [InlineData("1.01.1")] + [InlineData("1.00.0.1")] + [InlineData("1.0.0.0")] + [InlineData("1.0.01.0")] + public void GetNonhijackableUri_NormalizesPackagesVersion(string version) + { + // Arrange + var id = "abcd"; + var originalUri = new Uri($"https://www.nuget.org/api/v2/Packages(Id='{id}',Version='{version}')"); + var normalizedVersion = NuGetVersion.Parse(version).ToNormalizedString(); + + // Act + var newUri = UriUtils.GetNonhijackableUri(originalUri); + + // Assert + Assert.NotEqual(originalUri.ToString(), newUri.ToString()); + Assert.NotSame(originalUri, newUri); + + Assert.Equal($"https://www.nuget.org/api/v2/Packages?$filter=true and Id eq '{id}' and NormalizedVersion eq '{normalizedVersion}'&semVerLevel=2.0.0", newUri.ToString()); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/CatalogLeaf.cs b/tests/NgTests/Validation/CatalogLeaf.cs new file mode 100644 index 000000000..bc0a8a6f0 --- /dev/null +++ b/tests/NgTests/Validation/CatalogLeaf.cs @@ -0,0 +1,21 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Newtonsoft.Json; +using NuGet.Services.Metadata.Catalog; + +namespace NgTests.Validation +{ + public sealed class CatalogLeaf + { + [JsonProperty("created")] + public DateTimeOffset Created { get; set; } + [JsonProperty("lastEdited")] + public DateTimeOffset LastEdited { get; set; } + + [JsonProperty("packageEntries")] + public IEnumerable PackageEntries { get; set; } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/DummyAggregateValidator.cs b/tests/NgTests/Validation/DummyAggregateValidator.cs new file mode 100644 index 000000000..2828a1377 --- /dev/null +++ b/tests/NgTests/Validation/DummyAggregateValidator.cs @@ -0,0 +1,31 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Monitoring; + +namespace NgTests +{ + public class DummyAggregateValidator : IAggregateValidator + { + private IEnumerable _results; + + public string Name => nameof(DummyAggregateValidator); + + public DummyAggregateValidator(IEnumerable results) + { + _results = results; + } + + public AggregateValidationResult Validate() + { + return new AggregateValidationResult(this, _results); + } + + public Task ValidateAsync(ValidationContext context) + { + return Task.FromResult(Validate()); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/DummyValidator.cs b/tests/NgTests/Validation/DummyValidator.cs new file mode 100644 index 000000000..b8a9f28cc --- /dev/null +++ b/tests/NgTests/Validation/DummyValidator.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Monitoring; + +namespace NgTests +{ + public class DummyValidator : IValidator + { + private TestResult _result; + private Exception _exception; + + public string Name => nameof(DummyValidator); + + public DummyValidator(TestResult result, Exception e) + { + _result = result; + _exception = e; + } + + public Task ValidateAsync(ValidationContext context) + { + return Task.FromResult(Validate()); + } + + public ValidationResult Validate() + { + return new ValidationResult(this, _result, _exception); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/PackageHasSignatureValidatorFacts.cs b/tests/NgTests/Validation/PackageHasSignatureValidatorFacts.cs new file mode 100644 index 000000000..d8ce99782 --- /dev/null +++ b/tests/NgTests/Validation/PackageHasSignatureValidatorFacts.cs @@ -0,0 +1,343 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Versioning; +using Xunit; + +namespace NgTests.Validation +{ + public class PackageHasSignatureValidatorFacts + { + public class Constructor + { + private readonly CatalogEndpoint _endpoint; + private readonly ValidatorConfiguration _configuration; + + public Constructor() + { + _endpoint = ValidatorTestUtility.CreateCatalogEndpoint(); + _configuration = new ValidatorConfiguration(packageBaseAddress: "a", requireRepositorySignature: true); + } + + [Fact] + public void WhenEndpointIsNull_Throws() + { + var exception = Assert.Throws( + () => new PackageHasSignatureValidator( + endpoint: null, + config: _configuration, + logger: Mock.Of>())); + + Assert.Equal("endpoint", exception.ParamName); + } + + [Fact] + public void WhenConfigIsNull_Throws() + { + var exception = Assert.Throws( + () => new PackageHasSignatureValidator( + _endpoint, + config: null, + logger: Mock.Of>())); + + Assert.Equal("config", exception.ParamName); + } + + [Fact] + public void WhenLoggerIsNull_Throws() + { + var exception = Assert.Throws( + () => new PackageHasSignatureValidator( + _endpoint, + _configuration, + logger: null)); + + Assert.Equal("logger", exception.ParamName); + } + } + + public class ShouldRunValidator : FactsBase + { + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SkipsIfNoEntries(bool requirePackageSignature) + { + var target = CreateTarget(requirePackageSignature); + var context = CreateValidationContext(catalogEntries: new CatalogIndexEntry[0]); + + Assert.Equal(ShouldRunTestResult.No, target.ShouldRunValidator(context)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void SkipsIfLatestEntryIsDelete(bool requirePackageSignature) + { + var target = CreateTarget(requirePackageSignature); + var uri = new Uri($"https://nuget.test/{PackageIdentity.Id}"); + var context = CreateValidationContext( + catalogEntries: new[] + { + new CatalogIndexEntry( + uri, + type: CatalogConstants.NuGetPackageDetails, + commitId: Guid.NewGuid().ToString(), + commitTs: DateTime.MinValue, + packageIdentity: PackageIdentity), + new CatalogIndexEntry( + uri, + type: CatalogConstants.NuGetPackageDelete, + commitId: Guid.NewGuid().ToString(), + commitTs: DateTime.MinValue.AddDays(1), + packageIdentity: PackageIdentity), + }); + + Assert.Equal(ShouldRunTestResult.No, target.ShouldRunValidator(context)); + } + + [Fact] + public void SkipsIfLatestEntryIsNotDeleteAndPackageSignatureIsNotRequired() + { + var target = CreateTarget(requireRepositorySignature: false); + var context = CreateValidationContextWithLatestEntryWithDetails(); + Assert.Equal(ShouldRunTestResult.No, target.ShouldRunValidator(context)); + } + + [Fact] + public void RunsIfLatestEntryIsNotDeleteAndPackageSignatureIsRequired() + { + var target = CreateTarget(); + var context = CreateValidationContextWithLatestEntryWithDetails(); + Assert.Equal(ShouldRunTestResult.Yes, target.ShouldRunValidator(context)); + } + + private ValidationContext CreateValidationContextWithLatestEntryWithDetails() + { + var uri = new Uri($"https://nuget.test/{PackageIdentity.Id}"); + + return CreateValidationContext( + catalogEntries: new[] + { + new CatalogIndexEntry( + uri, + type: CatalogConstants.NuGetPackageDelete, + commitId: Guid.NewGuid().ToString(), + commitTs: DateTime.MinValue, + packageIdentity: PackageIdentity), + new CatalogIndexEntry( + uri, + type: CatalogConstants.NuGetPackageDetails, + commitId: Guid.NewGuid().ToString(), + commitTs: DateTime.MinValue.AddDays(1), + packageIdentity: PackageIdentity), + }); + } + } + + public class RunValidatorAsync : FactsBase + { + [Fact] + public async Task ReturnsGracefullyIfLatestLeafHasSignatureFile() + { + // Arrange + var target = CreateTarget(); + var context = CreateValidationContext( + catalogEntries: new[] + { + new CatalogIndexEntry( + uri: new Uri("https://nuget.test/a.json"), + type: CatalogConstants.NuGetPackageDetails, + commitId: Guid.NewGuid().ToString(), + commitTs: DateTime.MinValue, + packageIdentity: PackageIdentity), + new CatalogIndexEntry( + uri: new Uri("https://nuget.test/b.json"), + type: CatalogConstants.NuGetPackageDetails, + commitId: Guid.NewGuid().ToString(), + commitTs: DateTime.MinValue.AddDays(1), + packageIdentity: PackageIdentity), + }); + + AddCatalogLeaf("/a.json", new CatalogLeaf + { + PackageEntries = new[] + { + new PackageEntry { FullName = "hello.txt" } + } + }); + + AddCatalogLeaf("/b.json", new CatalogLeaf + { + PackageEntries = new[] + { + new PackageEntry { FullName = "hello.txt" }, + new PackageEntry { FullName = ".signature.p7s" } + } + }); + + // Act & Assert + await target.RunValidatorAsync(context); + } + + [Fact] + public async Task ThrowsIfLatestLeafIsMissingASignatureFile() + { + // Arrange + var malformedUri = new Uri("https://nuget.test/b.json"); + + var target = CreateTarget(); + var context = CreateValidationContext( + catalogEntries: new[] + { + new CatalogIndexEntry( + uri: new Uri("https://nuget.test/a.json"), + type: CatalogConstants.NuGetPackageDetails, + commitId: Guid.NewGuid().ToString(), + commitTs: DateTime.MinValue, + packageIdentity: PackageIdentity), + new CatalogIndexEntry( + uri: malformedUri, + type: CatalogConstants.NuGetPackageDetails, + commitId: Guid.NewGuid().ToString(), + commitTs: DateTime.MinValue.AddDays(1), + packageIdentity: PackageIdentity), + }); + + AddCatalogLeaf("/a.json", new CatalogLeaf + { + PackageEntries = new[] + { + new PackageEntry { FullName = ".signature.p7s" } + } + }); + + AddCatalogLeaf("/b.json", new CatalogLeaf + { + PackageEntries = new[] + { + new PackageEntry { FullName = "hello.txt" } + } + }); + + // Act & Assert + var e = await Assert.ThrowsAsync(() => target.RunValidatorAsync(context)); + + Assert.Same(malformedUri, e.CatalogEntry); + } + + [Fact] + public async Task ThrowsIfLeafPackageEntriesIsMissing() + { + // Arrange + var uri = new Uri("https://nuget.test/a.json"); + + var target = CreateTarget(); + var context = CreateValidationContext( + catalogEntries: new[] + { + new CatalogIndexEntry( + uri, + CatalogConstants.NuGetPackageDetails, + commitId: Guid.NewGuid().ToString(), + commitTs: DateTime.MinValue, + packageIdentity: PackageIdentity), + }); + + AddCatalogLeaf("/a.json", "{ 'this': 'is missing the packageEntries field' }"); + + // Act & Assert + var e = await Assert.ThrowsAsync(() => target.RunValidatorAsync(context)); + + Assert.Equal($"The catalog leaf at {uri.AbsoluteUri} is missing the 'packageEntries' property.", e.Message); + } + + [Fact] + public async Task ThrowsIfLeafPackageEntriesIsMalformed() + { + // Arrange + var uri = new Uri("https://nuget.test/a.json"); + + var target = CreateTarget(); + var context = CreateValidationContext( + catalogEntries: new[] + { + new CatalogIndexEntry( + uri, + CatalogConstants.NuGetPackageDetails, + commitId: Guid.NewGuid().ToString(), + commitTs: DateTime.MinValue, + packageIdentity: PackageIdentity), + }); + + AddCatalogLeaf("/a.json", "{ 'packageEntries': 'malformed' }"); + + // Act & Assert + var e = await Assert.ThrowsAsync(() => target.RunValidatorAsync(context)); + + Assert.Equal($"The catalog leaf at {uri.AbsoluteUri} has a malformed 'packageEntries' property.", e.Message); + } + } + + public class FactsBase + { + public static readonly PackageIdentity PackageIdentity = new PackageIdentity("TestPackage", NuGetVersion.Parse("1.0.0")); + + protected readonly Mock> _logger; + private readonly MockServerHttpClientHandler _mockServer; + + public FactsBase() + { + _logger = new Mock>(); + _mockServer = new MockServerHttpClientHandler(); + } + + protected ValidationContext CreateValidationContext(IEnumerable catalogEntries = null) + { + catalogEntries = catalogEntries ?? new CatalogIndexEntry[0]; + + var httpClient = new CollectorHttpClient(_mockServer); + + return ValidationContextStub.Create( + PackageIdentity, + catalogEntries, + client: httpClient); + } + + protected PackageHasSignatureValidator CreateTarget(bool requireRepositorySignature = true) + { + var endpoint = ValidatorTestUtility.CreateCatalogEndpoint(); + var config = ValidatorTestUtility.CreateValidatorConfig(requireRepositorySignature: requireRepositorySignature); + + return new PackageHasSignatureValidator(endpoint, config, _logger.Object); + } + + protected void AddCatalogLeaf(string path, CatalogLeaf leaf) + { + ValidatorTestUtility.AddCatalogLeafToMockServer(_mockServer, new Uri(path, UriKind.Relative), leaf); + } + + protected void AddCatalogLeaf(string path, string leafContent) + { + _mockServer.SetAction(path, request => + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(leafContent) + }); + }); + } + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/PackageIsRepositorySignedValidatorFacts.cs b/tests/NgTests/Validation/PackageIsRepositorySignedValidatorFacts.cs new file mode 100644 index 000000000..846fa936f --- /dev/null +++ b/tests/NgTests/Validation/PackageIsRepositorySignedValidatorFacts.cs @@ -0,0 +1,260 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Versioning; +using Xunit; + +namespace NgTests.Validation +{ + public class PackageIsRepositorySignedValidatorFacts + { + public class Constructor + { + private readonly ValidatorConfiguration _configuration; + private readonly FlatContainerEndpoint _endpoint; + + public Constructor() + { + _endpoint = ValidatorTestUtility.CreateFlatContainerEndpoint(); + _configuration = new ValidatorConfiguration(packageBaseAddress: "a", requireRepositorySignature: true); + } + + [Fact] + public void WhenEndpointIsNull_Throws() + { + var exception = Assert.Throws( + () => new PackageIsRepositorySignedValidator( + endpoint: null, + config: _configuration, + logger: Mock.Of>())); + + Assert.Equal("endpoint", exception.ParamName); + } + + [Fact] + public void WhenConfigIsNull_Throws() + { + var exception = Assert.Throws( + () => new PackageIsRepositorySignedValidator( + _endpoint, + config: null, + logger: Mock.Of>())); + + Assert.Equal("config", exception.ParamName); + } + + [Fact] + public void WhenLoggerIsNull_Throws() + { + var exception = Assert.Throws( + () => new PackageIsRepositorySignedValidator( + _endpoint, + _configuration, + logger: null)); + + Assert.Equal("logger", exception.ParamName); + } + } + + public class ValidateAsync : FactsBase + { + [Theory] + [InlineData((string)null)] + [InlineData(UnsignedPackageResource)] + [InlineData(AuthorSignedPackageResource)] + [InlineData(RepoSignedPackageResource)] + [InlineData(AuthorAndRepoSignedPackageResource)] + public async Task SkipsIfConfigDoesNotRequireRepositorySignature(string packageResource) + { + // Arrange + var target = CreateTarget(requireRepositorySignature: false); + var context = CreateValidationContext(packageResource); + + // Act + var result = await target.ValidateAsync(context); + + // Assert + Assert.Equal(TestResult.Skip, result.Result); + } + + [Fact] + public async Task SkipsIfPackageIsMissing() + { + // Arrange - modify the package ID on the validation context so that the + // nupkg can no longer be found. + var target = CreateTarget(); + var context = CreateValidationContext(packageResource: null); + + // Act + var result = await target.ValidateAsync(context); + + // Assert + Assert.Equal(TestResult.Skip, result.Result); + Assert.Null(result.Exception); + } + + [Fact] + public async Task FailsIfPackageHasNoSignature() + { + // Arrange + var target = CreateTarget(); + var context = CreateValidationContext(packageResource: UnsignedPackageResource); + + // Act + var result = await target.ValidateAsync(context); + + // Assert + var exception = result.Exception as MissingRepositorySignatureException; + + Assert.Equal(TestResult.Fail, result.Result); + Assert.NotNull(exception); + + Assert.Equal(MissingRepositorySignatureReason.Unsigned, exception.Reason); + Assert.StartsWith("Package TestPackage 1.0.0 is unsigned.", exception.Message); + } + + [Fact] + public async Task FailsIfPackageHasAnAuthorSignatureButNoRepositoryCountersignature() + { + // Arrange + var target = CreateTarget(); + var context = CreateValidationContext(AuthorSignedPackageResource); + + // Act + var result = await target.ValidateAsync(context); + + // Assert + var exception = result.Exception as MissingRepositorySignatureException; + + Assert.Equal(TestResult.Fail, result.Result); + Assert.NotNull(exception); + + Assert.Equal(MissingRepositorySignatureReason.AuthorSignedNoRepositoryCountersignature, exception.Reason); + Assert.StartsWith("Package TestPackage 1.0.0 is author signed but not repository signed.", exception.Message); + } + + [Fact] + public async Task PassesIfPackageHasARepositoryPrimarySignature() + { + // Arrange + var target = CreateTarget(); + var context = CreateValidationContext(packageResource: RepoSignedPackageResource); + + // Act + var result = await target.ValidateAsync(context); + + // Assert + Assert.Equal(TestResult.Pass, result.Result); + Assert.Null(result.Exception); + } + + [Fact] + public async Task PassesIfPackageHasARepositoryCountersignature() + { + // Arrange + var context = CreateValidationContext(packageResource: AuthorAndRepoSignedPackageResource); + + // Act + var target = CreateTarget(); + var result = await target.ValidateAsync(context); + + // Assert + Assert.Equal(TestResult.Pass, result.Result); + Assert.Null(result.Exception); + } + } + + public class FactsBase + { + public static readonly PackageIdentity PackageIdentity = new PackageIdentity("TestPackage", new NuGetVersion("1.0.0")); + + public const string UnsignedPackageResource = "Packages\\TestUnsigned.1.0.0.nupkg"; + public const string AuthorSignedPackageResource = "Packages\\TestSigned.leaf-1.1.0.0.nupkg"; + public const string RepoSignedPackageResource = "Packages\\TestRepoSigned.leaf-1.1.0.0.nupkg"; + public const string AuthorAndRepoSignedPackageResource = "Packages\\TestAuthorAndRepoSigned.leaf-1.1.0.0.nupkg"; + + public static readonly DateTime PackageCreationTime = DateTime.UtcNow; + + private readonly IEnumerable _catalogEntries; + private readonly MockServerHttpClientHandler _mockServer; + + public FactsBase() + { + _mockServer = new MockServerHttpClientHandler(); + + // Mock a catalog entry and leaf for the package we are validating. + _catalogEntries = new[] + { + new CatalogIndexEntry( + new Uri("https://nuget.test/catalog/leaf.json"), + CatalogConstants.NuGetPackageDetails, + Guid.NewGuid().ToString(), + DateTime.UtcNow, + PackageIdentity) + }; + + ValidatorTestUtility.AddCatalogLeafToMockServer(_mockServer, new Uri("/catalog/leaf.json", UriKind.Relative), new CatalogLeaf + { + Created = PackageCreationTime, + LastEdited = PackageCreationTime + }); + } + + protected PackageIsRepositorySignedValidator CreateTarget(bool requireRepositorySignature = true) + { + var endpoint = ValidatorTestUtility.CreateFlatContainerEndpoint(); + var logger = Mock.Of>(); + var config = ValidatorTestUtility.CreateValidatorConfig(requireRepositorySignature: requireRepositorySignature); + + return new PackageIsRepositorySignedValidator(endpoint, config, logger); + } + + protected ValidationContext CreateValidationContext(string packageResource = null) + { + // Add the package + if (packageResource != null) + { + var packageId = PackageIdentity.Id.ToLowerInvariant(); + var packageVersion = PackageIdentity.Version.ToNormalizedString().ToLowerInvariant(); + var relativeUrl = $"/packages/{packageId}/{packageVersion}/{packageId}.{packageVersion}.nupkg"; + var bytes = File.ReadAllBytes(packageResource); + + _mockServer.SetAction( + relativeUrl, + request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(bytes) + })); + } + + var httpClient = new CollectorHttpClient(_mockServer); + + // Mock V2 feed response for the package's Created/LastEdited timestamps. These timestamps must match + // the mocked catalog entry's timestamps. + var timestamp = PackageTimestampMetadata.CreateForExistingPackage(created: PackageCreationTime, lastEdited: PackageCreationTime); + var timestampMetadataResource = new Mock(); + + timestampMetadataResource.Setup(t => t.GetAsync(It.IsAny())) + .ReturnsAsync(timestamp); + + return ValidationContextStub.Create( + PackageIdentity, + _catalogEntries, + client: httpClient, + timestampMetadataResource: timestampMetadataResource.Object); + } + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/PackageRegistrationDeprecationMetadataTests.cs b/tests/NgTests/Validation/PackageRegistrationDeprecationMetadataTests.cs new file mode 100644 index 000000000..2b98fde35 --- /dev/null +++ b/tests/NgTests/Validation/PackageRegistrationDeprecationMetadataTests.cs @@ -0,0 +1,65 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Monitoring.Model; +using Xunit; + +namespace NgTests.Validation +{ + public class PackageRegistrationDeprecationMetadataTests + { + public class TheConstructorWithPackageDeprecationItem + { + [Fact] + public void SingleReason() + { + var deprecation = new PackageDeprecationItem(new[] { "a" }, null, null, null); + + var metadata = new PackageRegistrationDeprecationMetadata(deprecation); + + Assert.Equal(deprecation.Reasons, metadata.Reasons); + Assert.Null(metadata.Message); + Assert.Null(metadata.AlternatePackage); + } + + [Fact] + public void MultipleReasons() + { + var deprecation = new PackageDeprecationItem(new[] { "a", "b" }, null, null, null); + + var metadata = new PackageRegistrationDeprecationMetadata(deprecation); + + Assert.Equal(deprecation.Reasons, metadata.Reasons); + Assert.Null(metadata.Message); + Assert.Null(metadata.AlternatePackage); + } + + [Fact] + public void Message() + { + var deprecation = new PackageDeprecationItem(new[] { "c" }, "mmm", null, null); + + var metadata = new PackageRegistrationDeprecationMetadata(deprecation); + + Assert.Equal(deprecation.Reasons, metadata.Reasons); + Assert.Equal(deprecation.Message, metadata.Message); + Assert.Null(metadata.AlternatePackage); + } + + [Fact] + public void AlternatePackage() + { + var deprecation = new PackageDeprecationItem(new[] { "d" }, null, "abc", "cba"); + + var metadata = new PackageRegistrationDeprecationMetadata(deprecation); + + Assert.Equal(deprecation.Reasons, metadata.Reasons); + Assert.Null(metadata.Message); + Assert.NotNull(metadata.AlternatePackage); + Assert.Equal(deprecation.AlternatePackageId, metadata.AlternatePackage.Id); + Assert.Equal(deprecation.AlternatePackageRange, metadata.AlternatePackage.Range); + } + } + } +} diff --git a/tests/NgTests/Validation/PackageValidatorContextTests.cs b/tests/NgTests/Validation/PackageValidatorContextTests.cs new file mode 100644 index 000000000..a27ef455b --- /dev/null +++ b/tests/NgTests/Validation/PackageValidatorContextTests.cs @@ -0,0 +1,62 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Versioning; +using Xunit; + +namespace NgTests.Validation +{ + public class PackageValidatorContextTests + { + private static readonly PackageIdentity _packageIdentity = new PackageIdentity(id: "a", version: new NuGetVersion("1.0.0")); + private static readonly FeedPackageIdentity _feedPackageIdentity = new FeedPackageIdentity(_packageIdentity); + + [Fact] + public void Constructor_WhenPackageIsNull_Throws() + { + var exception = Assert.Throws( + () => new PackageValidatorContext( + package: null, + catalogEntries: Enumerable.Empty())); + + Assert.Equal("package", exception.ParamName); + } + + public static IEnumerable Constructor_WhenArgumentsAreValid_InitializesInstance_Data + { + get + { + yield return new object[] { null }; + + yield return new object[] { + new[] + { + new CatalogIndexEntry( + new Uri("https://nuget.test/a"), + CatalogConstants.NuGetPackageDetails, + Guid.NewGuid().ToString(), + DateTime.UtcNow, + _packageIdentity) + } + }; + } + } + + [Theory] + [MemberData(nameof(Constructor_WhenArgumentsAreValid_InitializesInstance_Data))] + public void Constructor_WhenArgumentsAreValid_InitializesInstance(CatalogIndexEntry[] catalogEntries) + { + var context = new PackageValidatorContext(_feedPackageIdentity, catalogEntries); + + Assert.Same(_feedPackageIdentity, context.Package); + Assert.Same(catalogEntries, context.CatalogEntries); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/PackageValidatorTests.cs b/tests/NgTests/Validation/PackageValidatorTests.cs new file mode 100644 index 000000000..575be32ba --- /dev/null +++ b/tests/NgTests/Validation/PackageValidatorTests.cs @@ -0,0 +1,238 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Protocol.Core.Types; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Versioning; +using Xunit; + +namespace NgTests.Validation +{ + public class PackageValidatorTests + { + private readonly List _validationResults; + private readonly DummyAggregateValidator _aggregateValidator; + private readonly IEnumerable _aggregateValidators; + private readonly ValidationSourceRepositories _sourceRepositories; + private readonly TestStorageFactory _storageFactory = new TestStorageFactory(); + private readonly ILogger _logger = Mock.Of>(); + private readonly ILogger _contextLogger = Mock.Of>(); + + public PackageValidatorTests() + { + _validationResults = new List(); + _aggregateValidator = new DummyAggregateValidator(_validationResults); + _aggregateValidators = new[] { _aggregateValidator }; + + _sourceRepositories = new ValidationSourceRepositories( + Mock.Of(), + Mock.Of()); + } + + [Fact] + public void Constructor_WhenAggregateValidatorsIsNull_Throws() + { + const IEnumerable aggregateValidators = null; + + var exception = Assert.Throws( + () => new PackageValidator( + aggregateValidators, + _storageFactory, + _sourceRepositories, + _logger, + _contextLogger)); + + Assert.Equal("aggregateValidators", exception.ParamName); + } + + [Fact] + public void Constructor_WhenAggregateValidatorsIsEmpty_Throws() + { + var aggregateValidators = Enumerable.Empty(); + + var exception = Assert.Throws( + () => new PackageValidator( + aggregateValidators, + _storageFactory, + _sourceRepositories, + _logger, + _contextLogger)); + + Assert.Equal("aggregateValidators", exception.ParamName); + } + + [Fact] + public void Constructor_WhenAuditingStorageFactoryIsNull_Throws() + { + const StorageFactory storageFactory = null; + + var exception = Assert.Throws( + () => new PackageValidator( + _aggregateValidators, + storageFactory, + _sourceRepositories, + _logger, + _contextLogger)); + + Assert.Equal("auditingStorageFactory", exception.ParamName); + } + + [Fact] + public void Constructor_WhenFeedToSourceIsNull_Throws() + { + const ValidationSourceRepositories sourceRepositories = null; + + var exception = Assert.Throws( + () => new PackageValidator( + _aggregateValidators, + _storageFactory, + sourceRepositories, + _logger, + _contextLogger)); + + Assert.Equal("sourceRepositories", exception.ParamName); + } + + [Fact] + public void Constructor_WhenContextLoggerIsNull_Throws() + { + const ILogger contextLogger = null; + + var exception = Assert.Throws( + () => new PackageValidator( + _aggregateValidators, + _storageFactory, + _sourceRepositories, + _logger, + contextLogger)); + + Assert.Equal("contextLogger", exception.ParamName); + } + + [Fact] + public void Constructor_WhenArgumentsAreValid_InitializesInstance() + { + var validator = new PackageValidator( + _aggregateValidators, + _storageFactory, + _sourceRepositories, + _logger, + _contextLogger); + + Assert.Equal(_aggregateValidators, validator.AggregateValidators); + } + + [Fact] + public async Task ValidateAsync_WhenContextIsNull_Throws() + { + var validator = new PackageValidator( + _aggregateValidators, + _storageFactory, + _sourceRepositories, + _logger, + _contextLogger); + + const PackageValidatorContext context = null; + + using (var client = new CollectorHttpClient()) + { + var exception = await Assert.ThrowsAsync( + () => validator.ValidateAsync(context, client, CancellationToken.None)); + + Assert.Equal("context", exception.ParamName); + } + } + + [Fact] + public async Task ValidateAsync_WhenClientIsNull_Throws() + { + var validator = new PackageValidator( + _aggregateValidators, + _storageFactory, + _sourceRepositories, + _logger, + _contextLogger); + + var context = new PackageValidatorContext( + new FeedPackageIdentity(id: "a", version: "1.0.0"), + catalogEntries: Enumerable.Empty()); + const CollectorHttpClient client = null; + + var exception = await Assert.ThrowsAsync( + () => validator.ValidateAsync(context, client, CancellationToken.None)); + + Assert.Equal("client", exception.ParamName); + } + + [Fact] + public async Task ValidateAsync_WhenCancellationTokenIsCancelled_Throws() + { + var validator = new PackageValidator( + _aggregateValidators, + _storageFactory, + _sourceRepositories, + _logger, + _contextLogger); + + var context = new PackageValidatorContext( + new FeedPackageIdentity(id: "a", version: "1.0.0"), + catalogEntries: Enumerable.Empty()); + + using (var client = new CollectorHttpClient()) + { + await Assert.ThrowsAsync( + () => validator.ValidateAsync(context, client, new CancellationToken(canceled: true))); + } + } + + [Fact] + public async Task ValidateAsync_WhenArgumentsAreValid_ReturnsResults() + { + var validator = new PackageValidator( + _aggregateValidators, + _storageFactory, + _sourceRepositories, + _logger, + _contextLogger); + + var packageIdentity = new PackageIdentity(id: "a", version: new NuGetVersion("1.0.0")); + var catalogEntries = new[] + { + new CatalogIndexEntry( + new Uri($"https://nuget.test/{packageIdentity.Id}"), + CatalogConstants.NuGetPackageDetails, + Guid.NewGuid().ToString(), + DateTime.UtcNow, + packageIdentity) + }; + var context = new PackageValidatorContext(new FeedPackageIdentity(packageIdentity), catalogEntries); + + using (var httpClient = new CollectorHttpClient()) + { + var actualResult = await validator.ValidateAsync(context, httpClient, CancellationToken.None); + + Assert.Equal(catalogEntries, actualResult.CatalogEntries); + Assert.Empty(actualResult.DeletionAuditEntries); + Assert.Equal(packageIdentity, actualResult.Package); + + Assert.Single(actualResult.AggregateValidationResults); + + var actualValidationResult = actualResult.AggregateValidationResults.Single(); + Assert.Same(_aggregateValidator, actualValidationResult.AggregateValidator); + Assert.Same(_validationResults, actualValidationResult.ValidationResults); + } + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/RegistrationDeprecationValidatorTestData.cs b/tests/NgTests/Validation/RegistrationDeprecationValidatorTestData.cs new file mode 100644 index 000000000..a90568c0e --- /dev/null +++ b/tests/NgTests/Validation/RegistrationDeprecationValidatorTestData.cs @@ -0,0 +1,139 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; +using NuGet.Services.Entities; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Services.Metadata.Catalog.Monitoring.Model; +using NuGet.Services.Metadata.Catalog.Monitoring.Validation.Test.Registration; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace NgTests.Validation +{ + public class RegistrationDeprecationValidatorTestData : RegistrationIndexValidatorTestData + { + protected override RegistrationDeprecationValidator CreateValidator( + ILogger logger) + { + var endpoint = ValidatorTestUtility.CreateRegistrationEndpoint(); + var config = new ValidatorConfiguration("https://nuget.test/packages", requireRepositorySignature: false); + + return new RegistrationDeprecationValidator(endpoint, config, logger); + } + + public override IEnumerable> CreateIndexes + { + get + { + yield return () => new PackageRegistrationIndexMetadata { Deprecation = null }; + + foreach (var reasons in GetPossibleDeprecationReasonCombinations()) + { + foreach (var hasMessage in new[] { false, true }) + { + yield return () => new PackageRegistrationIndexMetadata + { + Deprecation = new PackageRegistrationDeprecationMetadata + { + Reasons = reasons, + Message = hasMessage ? "this is the message" : null + } + }; + + yield return () => new PackageRegistrationIndexMetadata + { + Deprecation = new PackageRegistrationDeprecationMetadata + { + Reasons = reasons, + Message = hasMessage ? "this is the message" : null, + AlternatePackage = new PackageRegistrationAlternatePackageMetadata + { + Id = "thePackage", + Range = "*" + } + } + }; + + yield return () => new PackageRegistrationIndexMetadata + { + Deprecation = new PackageRegistrationDeprecationMetadata + { + Reasons = reasons, + Message = hasMessage ? "this is the message" : null, + AlternatePackage = new PackageRegistrationAlternatePackageMetadata + { + Id = "thePackage", + Range = "[1.0.0,1.0.0]" + } + } + }; + } + } + } + } + + public override IEnumerable>> CreateSpecialIndexes + { + get + { + // Multiple reasons can be in different orders + foreach (var reasons in GetPossibleDeprecationReasonCombinations().Where(c => c.Count() > 1)) + { + yield return () => Tuple.Create( + new PackageRegistrationIndexMetadata + { + Deprecation = new PackageRegistrationDeprecationMetadata + { + Reasons = reasons.OrderBy(r => r).ToList() + } + }, + new PackageRegistrationIndexMetadata + { + Deprecation = new PackageRegistrationDeprecationMetadata + { + Reasons = reasons.OrderByDescending(r => r).ToList() + } + }, + true); + } + } + } + + public override IEnumerable> CreateSkippedIndexes => new Func[] + { + () => null + }; + + private static IEnumerable> GetPossibleDeprecationReasonCombinations() + { + return GetNextPossibleDeprecationReasonCombinations( + Enum + .GetValues(typeof(PackageDeprecationStatus)) + .Cast() + .Where(s => s != PackageDeprecationStatus.NotDeprecated)); + } + + private static IEnumerable> GetNextPossibleDeprecationReasonCombinations(IEnumerable remainingStatuses) + { + if (!remainingStatuses.Any()) + { + yield break; + } + + var nextStatus = remainingStatuses.First().ToString(); + var nextPossibleStatuses = GetNextPossibleDeprecationReasonCombinations(remainingStatuses.Skip(1)); + if (nextPossibleStatuses.Any()) + { + foreach (var nextPossibleStatus in nextPossibleStatuses) + { + yield return nextPossibleStatus; + yield return nextPossibleStatus.Concat(new[] { nextStatus }); + } + } + + yield return new[] { nextStatus }; + } + } +} diff --git a/tests/NgTests/Validation/RegistrationExistsValidatorTestData.cs b/tests/NgTests/Validation/RegistrationExistsValidatorTestData.cs new file mode 100644 index 000000000..a578f5566 --- /dev/null +++ b/tests/NgTests/Validation/RegistrationExistsValidatorTestData.cs @@ -0,0 +1,72 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using NuGet.Services.Metadata.Catalog.Monitoring; + +namespace NgTests +{ + public class RegistrationExistsValidatorTestData : RegistrationLeafValidatorTestData + { + protected override RegistrationExistsValidator CreateValidator( + ILogger logger) + { + var endpoint = ValidatorTestUtility.CreateRegistrationEndpoint(); + var config = ValidatorTestUtility.CreateValidatorConfig(); + + return new RegistrationExistsValidator(endpoint, config, logger); + } + + public override IEnumerable> CreateIndexes => new Func[] + { + () => null + }; + + public override IEnumerable> CreateSkippedIndexes => new Func[0]; + + public override IEnumerable>> CreateSpecialIndexes => new Func>[] + { + () => Tuple.Create( + new PackageRegistrationIndexMetadata(), + new PackageRegistrationIndexMetadata(), + true), + + () => Tuple.Create( + null, + new PackageRegistrationIndexMetadata(), + false), + + () => Tuple.Create( + new PackageRegistrationIndexMetadata(), + null, + false) + }; + + public override IEnumerable> CreateLeafs => new Func[] + { + () => null + }; + + public override IEnumerable> CreateSkippedLeafs => new Func[0]; + + public override IEnumerable>> CreateSpecialLeafs => new Func>[] + { + () => Tuple.Create( + new PackageRegistrationLeafMetadata(), + new PackageRegistrationLeafMetadata(), + true), + + () => Tuple.Create( + null, + new PackageRegistrationLeafMetadata(), + true), + + () => Tuple.Create( + new PackageRegistrationLeafMetadata(), + null, + false) + }; + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/RegistrationIdValidatorTestData.cs b/tests/NgTests/Validation/RegistrationIdValidatorTestData.cs new file mode 100644 index 000000000..f39643dfa --- /dev/null +++ b/tests/NgTests/Validation/RegistrationIdValidatorTestData.cs @@ -0,0 +1,46 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using NuGet.Services.Metadata.Catalog.Monitoring; + +namespace NgTests +{ + public class RegistrationIdValidatorTestData : RegistrationIndexValidatorTestData + { + protected override RegistrationIdValidator CreateValidator( + ILogger logger) + { + var endpoint = ValidatorTestUtility.CreateRegistrationEndpoint(); + var config = ValidatorTestUtility.CreateValidatorConfig(); + + return new RegistrationIdValidator(endpoint, config, logger); + } + + public override IEnumerable> CreateIndexes => new Func[] + { + () => new PackageRegistrationIndexMetadata() { Id = "testPackage1" }, + () => new PackageRegistrationIndexMetadata() { Id = "testPackage2" } + }; + + public override IEnumerable> CreateSkippedIndexes => new Func[] + { + () => null + }; + + public override IEnumerable>> CreateSpecialIndexes => new Func>[] + { + () => Tuple.Create( + new PackageRegistrationIndexMetadata() { Id = "testPackage1" }, + new PackageRegistrationIndexMetadata() { Id = "testpackage1" }, + true), + + () => Tuple.Create( + new PackageRegistrationIndexMetadata() { Id = "testpackage1" }, + new PackageRegistrationIndexMetadata() { Id = "testPackage1" }, + true) + }; + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/RegistrationIndexValidatorTestData.cs b/tests/NgTests/Validation/RegistrationIndexValidatorTestData.cs new file mode 100644 index 000000000..3fe9eba64 --- /dev/null +++ b/tests/NgTests/Validation/RegistrationIndexValidatorTestData.cs @@ -0,0 +1,62 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using Moq; +using NuGet.Services.Metadata.Catalog.Monitoring; + +namespace NgTests +{ + public interface IRegistrationIndexValidatorTestData + { + /// + /// Creates the to run this data against. + /// + RegistrationIndexValidator CreateValidator(); + + /// + /// s to use for tests. + /// + /// This data must follow the following rules: + /// 1 - If one element is compared against the same element, it must succeed. + /// 2 - If one element is compared against a different element, it must fail. + /// 3 - Validation should always run against any pair of these elements. + /// + IEnumerable> CreateIndexes { get; } + + /// + /// s to use for tests. + /// + /// Validation should never run when any of these elements are included in a test. + /// + IEnumerable> CreateSkippedIndexes { get; } + + /// + /// s to use for tests. + /// + /// Each tuple of elements represents a pairing that should pass if true and fail otherwise. + /// + IEnumerable>> CreateSpecialIndexes { get; } + } + + public abstract class RegistrationIndexValidatorTestData : IRegistrationIndexValidatorTestData + where T : RegistrationIndexValidator + { + public RegistrationIndexValidator CreateValidator() + { + return CreateValidator(Mock.Of>()); + } + + protected abstract T CreateValidator(ILogger logger); + + public abstract IEnumerable> CreateIndexes { get; } + + public abstract IEnumerable> CreateSkippedIndexes { get; } + + public virtual IEnumerable>> CreateSpecialIndexes => + Enumerable.Empty>>(); + } +} diff --git a/tests/NgTests/Validation/RegistrationIndexValidatorTests.cs b/tests/NgTests/Validation/RegistrationIndexValidatorTests.cs new file mode 100644 index 000000000..19e64aecd --- /dev/null +++ b/tests/NgTests/Validation/RegistrationIndexValidatorTests.cs @@ -0,0 +1,129 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Monitoring; +using Xunit; + +namespace NgTests.Validation +{ + public class RegistrationIndexValidatorTests + { + private static IEnumerable ValidatorTestData(Func>> getPairs) + { + foreach (var testData in ValidatorTestUtility.GetImplementations()) + { + var validator = testData.CreateValidator(); + + foreach (var pair in getPairs(testData)) + { + yield return new object[] + { + validator, + pair.Item1, + pair.Item2 + }; + } + } + } + + private static IEnumerable ValidatorSpecialTestData(Func>> getPairs) + { + foreach (var testData in ValidatorTestUtility.GetImplementations()) + { + var validator = testData.CreateValidator(); + + foreach (var pair in getPairs(testData)) + { + yield return new object[] + { + validator, + pair.Item1, + pair.Item2, + pair.Item3 + }; + } + } + } + + public class TheCompareIndexMethod + { + public static IEnumerable ValidatorEqualIndexTestData => ValidatorTestData(t => ValidatorTestUtility.GetEqualPairs(t.CreateIndexes)); + + public static IEnumerable ValidatorUnequalIndexTestData => ValidatorTestData(t => ValidatorTestUtility.GetUnequalPairs(t.CreateIndexes)); + + public static IEnumerable ValidatorSpecialIndexTestData => ValidatorSpecialTestData(t => ValidatorTestUtility.GetSpecialPairs(t.CreateSpecialIndexes)); + + [Theory] + [MemberData(nameof(ValidatorEqualIndexTestData))] + public async Task PassesIfEqual( + RegistrationIndexValidator validator, + PackageRegistrationIndexMetadata v2, + PackageRegistrationIndexMetadata v3) + { + await validator.CompareIndexAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3); + } + + [Theory] + [MemberData(nameof(ValidatorUnequalIndexTestData))] + public async Task FailsIfUnequal( + RegistrationIndexValidator validator, + PackageRegistrationIndexMetadata v2, + PackageRegistrationIndexMetadata v3) + { + await Assert.ThrowsAnyAsync( + () => validator.CompareIndexAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + } + + [Theory] + [MemberData(nameof(ValidatorSpecialIndexTestData))] + public async Task SpecialCasesReturnAsExpected( + RegistrationIndexValidator validator, + PackageRegistrationIndexMetadata v2, + PackageRegistrationIndexMetadata v3, + bool shouldPass) + { + var compareTask = Task.Run(async () => await validator.CompareIndexAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + + if (shouldPass) + { + await compareTask; + } + else + { + await Assert.ThrowsAnyAsync( + () => compareTask); + } + } + } + + public class TheShouldRunIndexAsyncMethod + { + public static IEnumerable ValidatorRunIndexTestData => ValidatorTestData(t => ValidatorTestUtility.GetPairs(t.CreateIndexes)); + + public static IEnumerable ValidatorSkipIndexTestData => ValidatorTestData(t => ValidatorTestUtility.GetBigraphPairs(t.CreateIndexes, t.CreateSkippedIndexes)); + + [Theory] + [MemberData(nameof(ValidatorRunIndexTestData))] + public async Task Runs( + RegistrationIndexValidator validator, + PackageRegistrationIndexMetadata v2, + PackageRegistrationIndexMetadata v3) + { + Assert.Equal(ShouldRunTestResult.Yes, await validator.ShouldRunIndexAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + } + + [Theory] + [MemberData(nameof(ValidatorSkipIndexTestData))] + public async Task Skips( + RegistrationIndexValidator validator, + PackageRegistrationIndexMetadata v2, + PackageRegistrationIndexMetadata v3) + { + Assert.Equal(ShouldRunTestResult.No, await validator.ShouldRunIndexAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + } + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/RegistrationLeafValidatorTestData.cs b/tests/NgTests/Validation/RegistrationLeafValidatorTestData.cs new file mode 100644 index 000000000..06e79254c --- /dev/null +++ b/tests/NgTests/Validation/RegistrationLeafValidatorTestData.cs @@ -0,0 +1,93 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using Moq; +using NuGet.Services.Metadata.Catalog.Monitoring; + +namespace NgTests +{ + public interface IRegistrationLeafValidatorTestData + { + /// + /// Creates the to run this data against. + /// + RegistrationLeafValidator CreateValidator(); + + /// + /// s to use for tests. + /// + /// This data must follow the following rules: + /// 1 - If one element is compared against the same element, it must succeed. + /// 2 - If one element is compared against a different element, it must fail. + /// 3 - Validation should always run against any pair of these elements. + /// + IEnumerable> CreateIndexes { get; } + + /// + /// s to use for tests. + /// + /// Validation should never run when any of these elements are included in a test. + /// + IEnumerable> CreateSkippedIndexes { get; } + + /// + /// s to use for tests. + /// + /// Each tuple of elements represents a pairing that should pass if true and fail otherwise. + /// + IEnumerable>> CreateSpecialIndexes { get; } + + /// + /// s to use for tests. + /// + /// This data must follow the following rules: + /// 1 - If one element is compared against the same element, it must succeed. + /// 2 - If one element is compared against a different element, it must fail. + /// 3 - Validation should always run against any pair of these elements. + /// + IEnumerable> CreateLeafs { get; } + + /// + /// s to use for tests. + /// + /// Validation should never run when any of these elements are included in a test. + /// + IEnumerable> CreateSkippedLeafs { get; } + + /// + /// s to use for tests. + /// + /// Each tuple of elements represents a pairing that should pass if true and fail otherwise. + /// + IEnumerable>> CreateSpecialLeafs { get; } + } + + public abstract class RegistrationLeafValidatorTestData : IRegistrationLeafValidatorTestData + where T : RegistrationLeafValidator + { + public RegistrationLeafValidator CreateValidator() + { + return CreateValidator(Mock.Of>()); + } + + protected abstract T CreateValidator(ILogger logger); + + public abstract IEnumerable> CreateIndexes { get; } + + public abstract IEnumerable> CreateSkippedIndexes { get; } + + public virtual IEnumerable>> CreateSpecialIndexes => + Enumerable.Empty>>(); + + public abstract IEnumerable> CreateLeafs { get; } + + public abstract IEnumerable> CreateSkippedLeafs { get; } + + public virtual IEnumerable>> CreateSpecialLeafs => + Enumerable.Empty>>(); + } +} diff --git a/tests/NgTests/Validation/RegistrationLeafValidatorTests.cs b/tests/NgTests/Validation/RegistrationLeafValidatorTests.cs new file mode 100644 index 000000000..644113f0c --- /dev/null +++ b/tests/NgTests/Validation/RegistrationLeafValidatorTests.cs @@ -0,0 +1,213 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using NuGet.Services.Metadata.Catalog.Monitoring; +using Xunit; + +namespace NgTests.Validation +{ + public class RegistrationLeafValidatorTests + { + private static IEnumerable ValidatorTestData(Func>> getPairs) + { + foreach (var testData in ValidatorTestUtility.GetImplementations()) + { + var validator = testData.CreateValidator(); + + foreach (var pair in getPairs(testData)) + { + yield return new object[] + { + validator, + pair.Item1, + pair.Item2 + }; + } + } + } + + private static IEnumerable ValidatorSpecialTestData(Func>> getPairs) + { + foreach (var testData in ValidatorTestUtility.GetImplementations()) + { + var validator = testData.CreateValidator(); + + foreach (var pair in getPairs(testData)) + { + yield return new object[] + { + validator, + pair.Item1, + pair.Item2, + pair.Item3 + }; + } + } + } + + public class TheCompareLeafMethod + { + public class OnIndex + { + public static IEnumerable ValidatorEqualIndexTestData => ValidatorTestData(t => ValidatorTestUtility.GetEqualPairs(t.CreateIndexes)); + + public static IEnumerable ValidatorUnequalIndexTestData => ValidatorTestData(t => ValidatorTestUtility.GetUnequalPairs(t.CreateIndexes)); + + public static IEnumerable ValidatorSpecialIndexTestData => ValidatorSpecialTestData(t => ValidatorTestUtility.GetSpecialPairs(t.CreateSpecialIndexes)); + + [Theory] + [MemberData(nameof(ValidatorEqualIndexTestData))] + public async Task PassesIfEqual( + RegistrationLeafValidator validator, + PackageRegistrationIndexMetadata v2, + PackageRegistrationIndexMetadata v3) + { + await validator.CompareLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3); + } + + [Theory] + [MemberData(nameof(ValidatorUnequalIndexTestData))] + public async Task FailsIfUnequal( + RegistrationLeafValidator validator, + PackageRegistrationIndexMetadata v2, + PackageRegistrationIndexMetadata v3) + { + await Assert.ThrowsAnyAsync( + () => validator.CompareLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + } + + [Theory] + [MemberData(nameof(ValidatorSpecialIndexTestData))] + public async Task SpecialCasesReturnAsExpected( + RegistrationLeafValidator validator, + PackageRegistrationIndexMetadata v2, + PackageRegistrationIndexMetadata v3, + bool shouldPass) + { + var compareTask = Task.Run(async () => await validator.CompareLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + + if (shouldPass) + { + await compareTask; + } + else + { + await Assert.ThrowsAnyAsync( + () => compareTask); + } + } + } + + public class OnLeaf + { + public static IEnumerable ValidatorEqualLeafTestData => ValidatorTestData(t => ValidatorTestUtility.GetEqualPairs(t.CreateLeafs)); + + public static IEnumerable ValidatorUnequalLeafTestData => ValidatorTestData(t => ValidatorTestUtility.GetUnequalPairs(t.CreateLeafs)); + + public static IEnumerable ValidatorSpecialIndexTestData => ValidatorSpecialTestData(t => ValidatorTestUtility.GetSpecialPairs(t.CreateSpecialLeafs)); + + [Theory] + [MemberData(nameof(ValidatorEqualLeafTestData))] + public async Task PassesIfEqual( + RegistrationLeafValidator validator, + PackageRegistrationLeafMetadata v2, + PackageRegistrationLeafMetadata v3) + { + await validator.CompareLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3); + } + + [Theory] + [MemberData(nameof(ValidatorUnequalLeafTestData))] + public async Task FailsIfUnequal( + RegistrationLeafValidator validator, + PackageRegistrationLeafMetadata v2, + PackageRegistrationLeafMetadata v3) + { + await Assert.ThrowsAnyAsync( + () => validator.CompareLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + } + + [Theory] + [MemberData(nameof(ValidatorSpecialIndexTestData))] + public async Task SpecialCasesReturnAsExpected( + RegistrationLeafValidator validator, + PackageRegistrationLeafMetadata v2, + PackageRegistrationLeafMetadata v3, + bool shouldPass) + { + var compareTask = Task.Run(async () => await validator.CompareLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + + if (shouldPass) + { + await compareTask; + } + else + { + await Assert.ThrowsAnyAsync( + () => compareTask); + } + } + } + } + + public class TheShouldRunLeafMethod + { + public class OnIndex + { + public static IEnumerable ValidatorRunIndexTestData => ValidatorTestData(t => ValidatorTestUtility.GetPairs(t.CreateIndexes)); + + public static IEnumerable ValidatorSkipIndexTestData => ValidatorTestData(t => ValidatorTestUtility.GetBigraphPairs(t.CreateIndexes, t.CreateSkippedIndexes)); + + [Theory] + [MemberData(nameof(ValidatorRunIndexTestData))] + public async Task Runs( + RegistrationLeafValidator validator, + PackageRegistrationIndexMetadata v2, + PackageRegistrationIndexMetadata v3) + { + Assert.Equal(ShouldRunTestResult.Yes, await validator.ShouldRunLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + } + + [Theory] + [MemberData(nameof(ValidatorSkipIndexTestData))] + public async Task Skips( + RegistrationLeafValidator validator, + PackageRegistrationIndexMetadata v2, + PackageRegistrationIndexMetadata v3) + { + Assert.Equal(ShouldRunTestResult.No, await validator.ShouldRunLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + } + } + + public class OnLeaf + { + public static IEnumerable ValidatorRunLeafTestData => ValidatorTestData(t => ValidatorTestUtility.GetPairs(t.CreateLeafs)); + + public static IEnumerable ValidatorSkipLeafTestData => ValidatorTestData(t => ValidatorTestUtility.GetBigraphPairs(t.CreateLeafs, t.CreateSkippedLeafs)); + + [Theory] + [MemberData(nameof(ValidatorRunLeafTestData))] + public async Task Runs( + RegistrationLeafValidator validator, + PackageRegistrationLeafMetadata v2, + PackageRegistrationLeafMetadata v3) + { + Assert.Equal(ShouldRunTestResult.Yes, await validator.ShouldRunLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + } + + [Theory] + [MemberData(nameof(ValidatorSkipLeafTestData))] + public async Task Skips( + RegistrationLeafValidator validator, + PackageRegistrationLeafMetadata v2, + PackageRegistrationLeafMetadata v3) + { + Assert.Equal(ShouldRunTestResult.No, await validator.ShouldRunLeafAsync(ValidatorTestUtility.GetFakeValidationContext(), v2, v3)); + } + } + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/RegistrationListedValidatorTestData.cs b/tests/NgTests/Validation/RegistrationListedValidatorTestData.cs new file mode 100644 index 000000000..3e301b018 --- /dev/null +++ b/tests/NgTests/Validation/RegistrationListedValidatorTestData.cs @@ -0,0 +1,44 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using NuGet.Services.Metadata.Catalog.Monitoring; + +namespace NgTests +{ + public class RegistrationListedValidatorTestData : RegistrationLeafValidatorTestData + { + protected override RegistrationListedValidator CreateValidator( + ILogger logger) + { + var endpoint = ValidatorTestUtility.CreateRegistrationEndpoint(); + var config = ValidatorTestUtility.CreateValidatorConfig(); + + return new RegistrationListedValidator(endpoint, config, logger); + } + + public override IEnumerable> CreateIndexes => new Func[] + { + () => new PackageRegistrationIndexMetadata() { Listed = true }, + () => new PackageRegistrationIndexMetadata() { Listed = false } + }; + + public override IEnumerable> CreateSkippedIndexes => new Func[] + { + () => null + }; + + public override IEnumerable> CreateLeafs => new Func[] + { + () => new PackageRegistrationLeafMetadata() { Listed = true }, + () => new PackageRegistrationLeafMetadata() { Listed = false } + }; + + public override IEnumerable> CreateSkippedLeafs => new Func[] + { + () => null + }; + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/RegistrationRequireLicenseAcceptanceValidatorTestData.cs b/tests/NgTests/Validation/RegistrationRequireLicenseAcceptanceValidatorTestData.cs new file mode 100644 index 000000000..079bf0fa8 --- /dev/null +++ b/tests/NgTests/Validation/RegistrationRequireLicenseAcceptanceValidatorTestData.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using NuGet.Services.Metadata.Catalog.Monitoring; + +namespace NgTests +{ + public class RegistrationRequireLicenseAcceptanceValidatorTestData : RegistrationIndexValidatorTestData + { + protected override RegistrationRequireLicenseAcceptanceValidator CreateValidator( + ILogger logger) + { + var endpoint = ValidatorTestUtility.CreateRegistrationEndpoint(); + var config = new ValidatorConfiguration("https://nuget.test/packages", requireRepositorySignature: false); + + return new RegistrationRequireLicenseAcceptanceValidator(endpoint, config, logger); + } + + public override IEnumerable> CreateIndexes => new Func[] + { + () => new PackageRegistrationIndexMetadata() { RequireLicenseAcceptance = true }, + () => new PackageRegistrationIndexMetadata() { RequireLicenseAcceptance = false } + }; + + public override IEnumerable> CreateSkippedIndexes => new Func[] + { + () => null + }; + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/RegistrationVersionValidatorTestData.cs b/tests/NgTests/Validation/RegistrationVersionValidatorTestData.cs new file mode 100644 index 000000000..c24181dc0 --- /dev/null +++ b/tests/NgTests/Validation/RegistrationVersionValidatorTestData.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Versioning; + +namespace NgTests +{ + public class RegistrationVersionValidatorTestData : RegistrationIndexValidatorTestData + { + protected override RegistrationVersionValidator CreateValidator( + ILogger logger) + { + var endpoint = ValidatorTestUtility.CreateRegistrationEndpoint(); + var config = ValidatorTestUtility.CreateValidatorConfig(); + + return new RegistrationVersionValidator(endpoint, config, logger); + } + + public override IEnumerable> CreateIndexes => new Func[] + { + () => new PackageRegistrationIndexMetadata() { Version = new NuGetVersion("1.0.0") }, + () => new PackageRegistrationIndexMetadata() { Version = new NuGetVersion("2.0.0-pre") }, + () => new PackageRegistrationIndexMetadata() { Version = new NuGetVersion("3.4.3+build") }, + () => new PackageRegistrationIndexMetadata() { Version = new NuGetVersion("87.23.11-alpha.9") } + }; + + public override IEnumerable> CreateSkippedIndexes => new Func[] + { + () => null + }; + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/SearchHasVersionValidatorFacts.cs b/tests/NgTests/Validation/SearchHasVersionValidatorFacts.cs new file mode 100644 index 000000000..c620c99dc --- /dev/null +++ b/tests/NgTests/Validation/SearchHasVersionValidatorFacts.cs @@ -0,0 +1,308 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using Newtonsoft.Json; +using NgTests.Infrastructure; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Versioning; +using Xunit; + +namespace NgTests.Validation +{ + public class SearchHasVersionValidatorFacts + { + public class ValidateAsync : FactsBase + { + [Fact] + public async Task FailsIfPackageIsListedInDatabaseButUnlistedInSearch() + { + // Arrange + var target = CreateTarget(); + var context = CreateValidationContext(); + DatabaseIndex = new PackageRegistrationIndexMetadata { Listed = true }; + MakeEmptyResultInSearch(); + + // Act + var result = await target.ValidateAsync(context); + + // Assert + Assert.Equal(TestResult.Fail, result.Result); + var exception = Assert.IsType(result.Exception); + Assert.Equal( + "The metadata between the database and V3 is inconsistent! Database shows listed but search shows unlisted.", + exception.Message); + } + + [Fact] + public async Task FailsIfPackageIsListedInDatabaseButOtherVersionIsListedInSearch() + { + // Arrange + var target = CreateTarget(); + var context = CreateValidationContext(); + DatabaseIndex = new PackageRegistrationIndexMetadata { Listed = true }; + MakeVersionVisibleInSearch(OtherVersion); + + // Act + var result = await target.ValidateAsync(context); + + // Assert + Assert.Equal(TestResult.Fail, result.Result); + var exception = Assert.IsType(result.Exception); + Assert.Equal( + "The metadata between the database and V3 is inconsistent! Database shows listed but search shows unlisted.", + exception.Message); + } + + [Fact] + public async Task FailsIfPackageIsUnavailableInDatabaseButListedInSearch() + { + // Arrange + var target = CreateTarget(); + var context = CreateValidationContext(); + DatabaseIndex = null; + MakeVersionVisibleInSearch(); + + // Act + var result = await target.ValidateAsync(context); + + // Assert + Assert.Equal(TestResult.Fail, result.Result); + var exception = Assert.IsType(result.Exception); + Assert.Equal( + "The metadata between the database and V3 is inconsistent! Database shows unavailable but search shows listed.", + exception.Message); + } + + [Fact] + public async Task FailsIfPackageIsUnlistedInDatabaseButListedInSearch() + { + // Arrange + var target = CreateTarget(); + var context = CreateValidationContext(); + DatabaseIndex = new PackageRegistrationIndexMetadata { Listed = false }; + MakeVersionVisibleInSearch(); + + // Act + var result = await target.ValidateAsync(context); + + // Assert + Assert.Equal(TestResult.Fail, result.Result); + var exception = Assert.IsType(result.Exception); + Assert.Equal( + "The metadata between the database and V3 is inconsistent! Database shows unlisted but search shows listed.", + exception.Message); + } + + [Fact] + public async Task SucceedsIfPackageIsUnavailableInDatabaseAndOtherVersionInSearch() + { + // Arrange + var target = CreateTarget(); + var context = CreateValidationContext(); + DatabaseIndex = null; + MakeVersionVisibleInSearch(OtherVersion); + + // Act + var result = await target.ValidateAsync(context); + + // Assert + Assert.Equal(TestResult.Pass, result.Result); + } + + [Fact] + public async Task SucceedsIfPackageIsUnavailableInDatabaseAndUnlistedInSearch() + { + // Arrange + var target = CreateTarget(); + var context = CreateValidationContext(); + DatabaseIndex = null; + MakeEmptyResultInSearch(); + + // Act + var result = await target.ValidateAsync(context); + + // Assert + Assert.Equal(TestResult.Pass, result.Result); + } + + [Fact] + public async Task SucceedsIfPackageIsUnlistedInDatabaseAndOtherVersionInSearch() + { + // Arrange + var target = CreateTarget(); + var context = CreateValidationContext(); + DatabaseIndex = new PackageRegistrationIndexMetadata { Listed = false }; + MakeEmptyResultInSearch(OtherVersion); + + // Act + var result = await target.ValidateAsync(context); + + // Assert + Assert.Equal(TestResult.Pass, result.Result); + } + + [Fact] + public async Task SucceedsIfPackageIsUnlistedInDatabaseAndUnlistedInSearch() + { + // Arrange + var target = CreateTarget(); + var context = CreateValidationContext(); + DatabaseIndex = new PackageRegistrationIndexMetadata { Listed = false }; + MakeEmptyResultInSearch(); + + // Act + var result = await target.ValidateAsync(context); + + // Assert + Assert.Equal(TestResult.Pass, result.Result); + } + + [Fact] + public async Task SucceedsIfPackageIsListedInDatabaseAndListedInSearch() + { + // Arrange + var target = CreateTarget(); + var context = CreateValidationContext(); + DatabaseIndex = new PackageRegistrationIndexMetadata { Listed = true }; + MakeVersionVisibleInSearch(); + + // Act + var result = await target.ValidateAsync(context); + + // Assert + Assert.Equal(TestResult.Pass, result.Result); + } + } + + public class FactsBase + { + public const string OtherVersion = "9.9.9"; + public static readonly PackageIdentity PackageIdentity = new PackageIdentity("TestPackage", new NuGetVersion("1.0.0")); + public static readonly DateTime PackageCreationTime = DateTime.UtcNow; + + public CatalogIndexEntry[] CatalogEntries { get; } + public MockServerHttpClientHandler MockServer { get; } + public PackageRegistrationIndexMetadata DatabaseIndex { get; set; } + + public FactsBase() + { + MockServer = new MockServerHttpClientHandler(); + + // Mock a catalog entry and leaf for the package we are validating. + CatalogEntries = new[] + { + new CatalogIndexEntry( + new Uri("https://nuget.test/catalog/leaf.json"), + CatalogConstants.NuGetPackageDetails, + Guid.NewGuid().ToString(), + DateTime.UtcNow, + PackageIdentity) + }; + + ValidatorTestUtility.AddCatalogLeafToMockServer(MockServer, new Uri("/catalog/leaf.json", UriKind.Relative), new CatalogLeaf + { + Created = PackageCreationTime, + LastEdited = PackageCreationTime + }); + } + + public void MakeVersionVisibleInSearch( + string version = null) + { + MockServer.SetAction( + $"/query?q=packageid:{PackageIdentity.Id}&skip=0&take=1&prerelease=true&semVerLevel=2.0.0", + request => + { + var json = JsonConvert.SerializeObject(new + { + data = new[] + { + new + { + id = PackageIdentity.Id, + version = version ?? PackageIdentity.Version.ToNormalizedString(), + versions = new[] + { + new + { + version = version ?? PackageIdentity.Version.ToNormalizedString(), + downloads = 0, + } + } + } + } + }); + + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }; + + return Task.FromResult(response); + }); + } + + public void MakeEmptyResultInSearch( + string version = null) + { + MockServer.SetAction( + $"/query?q=packageid:{PackageIdentity.Id}&skip=0&take=1&prerelease=true&semVerLevel=2.0.0", + request => + { + var json = JsonConvert.SerializeObject(new + { + data = new object[0] + }); + + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }; + + return Task.FromResult(response); + }); + } + + protected SearchHasVersionValidator CreateTarget() + { + var endpoint = ValidatorTestUtility.CreateSearchEndpoint(); + var logger = Mock.Of>(); + var config = ValidatorTestUtility.CreateValidatorConfig(); + + return new SearchHasVersionValidator(endpoint, config, logger); + } + + protected ValidationContext CreateValidationContext() + { + var timestamp = PackageTimestampMetadata.CreateForExistingPackage(created: PackageCreationTime, lastEdited: PackageCreationTime); + var timestampMetadataResource = new Mock(); + timestampMetadataResource.Setup(t => t.GetAsync(It.IsAny())).ReturnsAsync(timestamp); + + var v2Resource = new Mock(); + v2Resource + .Setup(x => x.GetIndexAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(() => DatabaseIndex); + + return ValidationContextStub.Create( + PackageIdentity, + CatalogEntries, + clientHandler: MockServer, + timestampMetadataResource: timestampMetadataResource.Object, + v2Resource: v2Resource.Object); + } + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/ValidationContextStub.cs b/tests/NgTests/Validation/ValidationContextStub.cs new file mode 100644 index 000000000..cc74b57d8 --- /dev/null +++ b/tests/NgTests/Validation/ValidationContextStub.cs @@ -0,0 +1,93 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NuGet.Configuration; +using NuGet.Packaging.Core; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Versioning; + +namespace NgTests.Validation +{ + internal sealed class ValidationContextStub : ValidationContext + { + private static readonly PackageIdentity _packageIdentity = new PackageIdentity("A", new NuGetVersion(1, 0, 0)); + + private ValidationContextStub( + PackageIdentity package, + IEnumerable entries, + IEnumerable deletionAuditEntries, + ValidationSourceRepositories sourceRepositories, + CollectorHttpClient client, + CancellationToken token, + ILogger logger) + : base(package, entries, deletionAuditEntries, sourceRepositories, client, token, logger) + { + } + + internal static ValidationContextStub Create( + PackageIdentity package = null, + IEnumerable entries = null, + IEnumerable deletionAuditEntries = null, + Dictionary feedToSource = null, + CollectorHttpClient client = null, + HttpClientHandler clientHandler = null, + CancellationToken? token = null, + ILogger logger = null, + IPackageTimestampMetadataResource timestampMetadataResource = null, + IPackageRegistrationMetadataResource v2Resource = null, + IPackageRegistrationMetadataResource v3Resource = null) + { + if (feedToSource == null) + { + feedToSource = new Dictionary(); + + var v2Repository = new Mock(); + var v3Repository = new Mock(); + + feedToSource.Add(FeedType.HttpV2, v2Repository.Object); + feedToSource.Add(FeedType.HttpV3, v3Repository.Object); + + v2Repository.Setup(x => x.GetResource()) + .Returns(timestampMetadataResource ?? Mock.Of()); + + v2Repository.Setup(x => x.GetResource()) + .Returns(v2Resource ?? Mock.Of()); + + v3Repository.Setup(x => x.GetResource()) + .Returns(v3Resource ?? Mock.Of()); + + v3Repository.Setup(x => x.GetResource()) + .Returns(new HttpSourceResource(new HttpSource( + new PackageSource("https://example/v3/index.json"), + () => Task.FromResult(new HttpHandlerResourceV3( + clientHandler ?? new HttpClientHandler(), + clientHandler ?? Mock.Of())), + NullThrottle.Instance))); + } + + var sourceRepositories = new ValidationSourceRepositories( + feedToSource[FeedType.HttpV2], + feedToSource[FeedType.HttpV3]); + + return new ValidationContextStub( + package ?? _packageIdentity, + entries ?? Enumerable.Empty(), + deletionAuditEntries ?? Enumerable.Empty(), + sourceRepositories, + client ?? new CollectorHttpClient(clientHandler), + token ?? CancellationToken.None, + logger ?? Mock.Of>()); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/ValidationContextTests.cs b/tests/NgTests/Validation/ValidationContextTests.cs new file mode 100644 index 000000000..4e82655e9 --- /dev/null +++ b/tests/NgTests/Validation/ValidationContextTests.cs @@ -0,0 +1,356 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NuGet.Packaging.Core; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Versioning; +using Xunit; + +namespace NgTests.Validation +{ + public class ValidationContextTests + { + private static readonly PackageIdentity _packageIdentity = new PackageIdentity("A", new NuGetVersion(1, 0, 0)); + private static readonly ValidationSourceRepositories _mockValidationSourceRepositories = + new ValidationSourceRepositories(Mock.Of(), Mock.Of()); + + [Fact] + public void Constructor_WhenPackageIdentityIsNull_Throws() + { + PackageIdentity packageIdentity = null; + + var exception = Assert.Throws( + () => new ValidationContext( + packageIdentity, + Enumerable.Empty(), + Enumerable.Empty(), + _mockValidationSourceRepositories, + new CollectorHttpClient(), + CancellationToken.None, + Mock.Of>())); + + Assert.Equal("package", exception.ParamName); + } + + [Fact] + public void Constructor_WhenDeletionAuditEntriesIsNull_Throws() + { + IEnumerable deletionAuditEntries = null; + + var exception = Assert.Throws( + () => new ValidationContext( + _packageIdentity, + Enumerable.Empty(), + deletionAuditEntries, + _mockValidationSourceRepositories, + new CollectorHttpClient(), + CancellationToken.None, + Mock.Of>())); + + Assert.Equal("deletionAuditEntries", exception.ParamName); + } + + [Fact] + public void Constructor_WhenFeedToSourceIsNull_Throws() + { + ValidationSourceRepositories sourceRepositories = null; + + var exception = Assert.Throws( + () => new ValidationContext( + _packageIdentity, + Enumerable.Empty(), + Enumerable.Empty(), + sourceRepositories, + new CollectorHttpClient(), + CancellationToken.None, + Mock.Of>())); + + Assert.Equal("sourceRepositories", exception.ParamName); + } + + [Fact] + public void Constructor_WhenClientIsNull_Throws() + { + CollectorHttpClient client = null; + + var exception = Assert.Throws( + () => new ValidationContext( + _packageIdentity, + Enumerable.Empty(), + Enumerable.Empty(), + _mockValidationSourceRepositories, + client, + CancellationToken.None, + Mock.Of>())); + + Assert.Equal("client", exception.ParamName); + } + + [Fact] + public void Constructor_WhenLoggerIsNull_Throws() + { + var exception = Assert.Throws( + () => new ValidationContext( + _packageIdentity, + Enumerable.Empty(), + Enumerable.Empty(), + _mockValidationSourceRepositories, + new CollectorHttpClient(), + CancellationToken.None, + logger: null)); + + Assert.Equal("logger", exception.ParamName); + } + + [Fact] + public void Package_ReturnsCorrectValue() + { + var context = CreateContext(_packageIdentity); + + Assert.Same(_packageIdentity, context.Package); + } + + [Fact] + public void Entries_ReturnsCorrectValue() + { + var entry = new CatalogIndexEntry( + new Uri("https://nuget.test"), + CatalogConstants.NuGetPackageDetails, + Guid.NewGuid().ToString(), + DateTime.UtcNow, + _packageIdentity); + var entries = new[] { entry }; + var context = CreateContext(entries: entries); + + Assert.Equal(entries.Length, context.Entries.Count); + Assert.Same(entry, context.Entries[0]); + } + + [Fact] + public void DeletionAuditEntries_ReturnsCorrectValue() + { + var entry = new DeletionAuditEntry(); + var entries = new[] { entry }; + var context = CreateContext(deletionAuditEntries: entries); + + Assert.Equal(entries.Length, context.DeletionAuditEntries.Count); + Assert.Same(entry, context.DeletionAuditEntries[0]); + } + + [Fact] + public void Client_ReturnsCorrectValue() + { + var client = new CollectorHttpClient(); + var context = CreateContext(client: client); + + Assert.Same(client, context.Client); + } + + [Fact] + public void CancellationToken_ReturnsCorrectValue() + { + var token = new CancellationToken(canceled: true); + var context = CreateContext(token: token); + + Assert.Equal(token, context.CancellationToken); + } + + [Fact] + public void GetIndexV2Async_ReturnsMemoizedIndexTask() + { + var context = CreateContext(); + + var task1 = context.GetIndexDatabaseAsync(); + var task2 = context.GetIndexDatabaseAsync(); + + Assert.Same(task1, task2); + } + + [Fact] + public async Task GetIndexV2Async_ReturnsIndex() + { + var v2Resource = new Mock(); + var expectedResult = Mock.Of(); + + v2Resource.Setup(x => x.GetIndexAsync(It.IsNotNull(), It.IsNotNull(), It.IsAny())) + .ReturnsAsync(expectedResult); + + var context = CreateContext(v2Resource: v2Resource.Object); + + var actualResult = await context.GetIndexDatabaseAsync(); + + Assert.Same(expectedResult, actualResult); + } + + [Fact] + public void GetIndexV3Async_ReturnsMemoizedIndexTask() + { + var context = CreateContext(); + + var task1 = context.GetIndexV3Async(); + var task2 = context.GetIndexV3Async(); + + Assert.Same(task1, task2); + } + + [Fact] + public async Task GetIndexV3Async_ReturnsIndex() + { + var v3Resource = new Mock(); + var expectedResult = Mock.Of(); + + v3Resource.Setup(x => x.GetIndexAsync(It.IsNotNull(), It.IsNotNull(), It.IsAny())) + .ReturnsAsync(expectedResult); + + var context = CreateContext(v3Resource: v3Resource.Object); + + var actualResult = await context.GetIndexV3Async(); + + Assert.Same(expectedResult, actualResult); + } + + [Fact] + public void GetLeafV2Async_ReturnsMemoizedIndexTask() + { + var context = CreateContext(); + + var task1 = context.GetLeafDatabaseAsync(); + var task2 = context.GetLeafDatabaseAsync(); + + Assert.Same(task1, task2); + } + + [Fact] + public async Task GetLeafV2Async_ReturnsLeaf() + { + var v2Resource = new Mock(); + var expectedResult = Mock.Of(); + + v2Resource.Setup(x => x.GetLeafAsync(It.IsNotNull(), It.IsNotNull(), It.IsAny())) + .ReturnsAsync(expectedResult); + + var context = CreateContext(v2Resource: v2Resource.Object); + + var actualResult = await context.GetLeafDatabaseAsync(); + + Assert.Same(expectedResult, actualResult); + } + + [Fact] + public void GetLeafV3Async_ReturnsMemoizedIndexTask() + { + var context = CreateContext(); + + var task1 = context.GetLeafV3Async(); + var task2 = context.GetLeafV3Async(); + + Assert.Same(task1, task2); + } + + [Fact] + public async Task GetLeafV3Async_ReturnsLeaf() + { + var v3Resource = new Mock(); + var expectedResult = Mock.Of(); + + v3Resource.Setup(x => x.GetLeafAsync(It.IsNotNull(), It.IsNotNull(), It.IsAny())) + .ReturnsAsync(expectedResult); + + var context = CreateContext(v3Resource: v3Resource.Object); + + var actualResult = await context.GetLeafV3Async(); + + Assert.Same(expectedResult, actualResult); + } + + [Fact] + public async Task GetTimestampMetadataV2Async_ReturnsCorrectValue() + { + var timestampMetadataResourceV2 = new Mock(); + var timestampMetadata = new PackageTimestampMetadata(); + + var context = CreateContext(timestampMetadataResource: timestampMetadataResourceV2.Object); + + timestampMetadataResourceV2.Setup(x => x.GetAsync(It.Is(vc => vc == context))) + .ReturnsAsync(timestampMetadata); + + var actualResult = await context.GetTimestampMetadataDatabaseAsync(); + + Assert.Same(timestampMetadata, actualResult); + } + + [Fact] + public void GetTimestampMetadataV2Async_ReturnsMemoizedTimestampMetadataV2Task() + { + var timestampMetadataResourceV2 = new Mock(); + var timestampMetadata = new PackageTimestampMetadata(); + + var context = CreateContext(timestampMetadataResource: timestampMetadataResourceV2.Object); + + timestampMetadataResourceV2.Setup(x => x.GetAsync(It.Is(vc => vc == context))) + .ReturnsAsync(timestampMetadata); + + var task1 = context.GetTimestampMetadataDatabaseAsync(); + var task2 = context.GetTimestampMetadataDatabaseAsync(); + + Assert.Same(task1, task2); + } + + private static ValidationContext CreateContext( + PackageIdentity package = null, + IEnumerable entries = null, + IEnumerable deletionAuditEntries = null, + Dictionary feedToSource = null, + CollectorHttpClient client = null, + CancellationToken? token = null, + ILogger logger = null, + IPackageTimestampMetadataResource timestampMetadataResource = null, + IPackageRegistrationMetadataResource v2Resource = null, + IPackageRegistrationMetadataResource v3Resource = null) + { + if (feedToSource == null) + { + feedToSource = new Dictionary(); + + var v2Repository = new Mock(); + var v3Repository = new Mock(); + + feedToSource.Add(FeedType.HttpV2, v2Repository.Object); + feedToSource.Add(FeedType.HttpV3, v3Repository.Object); + + v2Repository.Setup(x => x.GetResource()) + .Returns(timestampMetadataResource ?? Mock.Of()); + + v2Repository.Setup(x => x.GetResource()) + .Returns(v2Resource ?? Mock.Of()); + + v3Repository.Setup(x => x.GetResource()) + .Returns(v3Resource ?? Mock.Of()); + } + + var sourceRepositories = new ValidationSourceRepositories( + feedToSource[FeedType.HttpV2], + feedToSource[FeedType.HttpV3]); + + return new ValidationContext( + package ?? _packageIdentity, + entries ?? Enumerable.Empty(), + deletionAuditEntries ?? Enumerable.Empty(), + sourceRepositories, + client ?? new CollectorHttpClient(), + token ?? CancellationToken.None, + logger ?? Mock.Of>()); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/ValidatorConfigurationTests.cs b/tests/NgTests/Validation/ValidatorConfigurationTests.cs new file mode 100644 index 000000000..a9e440963 --- /dev/null +++ b/tests/NgTests/Validation/ValidatorConfigurationTests.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Services.Metadata.Catalog.Monitoring; +using Xunit; + +namespace NgTests.Validation +{ + public class ValidatorConfigurationTests + { + [Theory] + [InlineData(null)] + [InlineData("")] + public void Constructor_WhenPackageBaseAddressIsNullOrEmpty_Throws(string packageBaseAddress) + { + var exception = Assert.Throws( + () => new ValidatorConfiguration(packageBaseAddress, requireRepositorySignature: true)); + + Assert.Equal("packageBaseAddress", exception.ParamName); + } + + [Theory] + [InlineData("a", true)] + [InlineData("b", false)] + public void Constructor_WhenArgumentsAreValid_InitializesInstance( + string packageBaseAddress, + bool requireRepositorySignature) + { + var configuration = new ValidatorConfiguration(packageBaseAddress, requireRepositorySignature); + + Assert.Equal(packageBaseAddress, configuration.PackageBaseAddress); + Assert.Equal(requireRepositorySignature, configuration.RequireRepositorySignature); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/ValidatorTestUtility.cs b/tests/NgTests/Validation/ValidatorTestUtility.cs new file mode 100644 index 000000000..859428b4a --- /dev/null +++ b/tests/NgTests/Validation/ValidatorTestUtility.cs @@ -0,0 +1,176 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Reflection; +using System.Threading.Tasks; +using Newtonsoft.Json; +using NgTests.Infrastructure; +using NgTests.Validation; +using NuGet.Packaging.Core; +using NuGet.Services.Metadata.Catalog.Monitoring; +using NuGet.Versioning; + +namespace NgTests +{ + public static class ValidatorTestUtility + { + public static EndpointConfiguration CreateEndpointConfig() + { + return new EndpointConfiguration( + registrationCursorUri: new Uri("https://example/reg"), + flatContainerCursorUri: new Uri("https://example/fc"), + instanceNameToSearchConfig: new Dictionary()); + } + + public static CatalogEndpoint CreateCatalogEndpoint() + { + return new CatalogEndpoint(); + } + + public static FlatContainerEndpoint CreateFlatContainerEndpoint() + { + return new FlatContainerEndpoint( + CreateEndpointConfig(), + () => null); + } + + public static RegistrationEndpoint CreateRegistrationEndpoint() + { + return new RegistrationEndpoint( + CreateEndpointConfig(), + () => null); + } + + public static SearchEndpoint CreateSearchEndpoint( + string instanceName = "usnc", + string cursorUri = "https://example/search/cursor.json", + string baseUri = "https://example-search/") + { + return new SearchEndpoint( + instanceName, + new[] { new Uri(cursorUri) }, + new Uri(baseUri), + () => null); + } + + public static ValidatorConfiguration CreateValidatorConfig( + string packageBaseAddress = "https://nuget.test/packages", + bool requireRepositorySignature = false) + { + return new ValidatorConfiguration(packageBaseAddress, requireRepositorySignature); + } + + public static IEnumerable> GetPairs(IEnumerable> valueFactories) + { + var set = valueFactories; + for (var factoryI = 0; factoryI < set.Count(); factoryI++) + { + for (var factoryJ = 0; factoryJ < set.Count(); factoryJ++) + { + yield return Tuple.Create(set.ElementAt(factoryI)(), set.ElementAt(factoryJ)()); + } + } + } + + public static IEnumerable> GetBigraphPairs(IEnumerable> valueFactories1, IEnumerable> valueFactories2) + { + return GetOneSidedBigraphPairs(valueFactories1, valueFactories2) + .Concat(GetOneSidedBigraphPairs(valueFactories2, valueFactories1)); + } + + private static IEnumerable> GetOneSidedBigraphPairs(IEnumerable> valueFactories1, IEnumerable> valueFactories2) + { + for (var factoryI = 0; factoryI < valueFactories1.Count(); factoryI++) + { + for (var factoryJ = 0; factoryJ < valueFactories2.Count(); factoryJ++) + { + yield return Tuple.Create(valueFactories1.ElementAt(factoryI)(), valueFactories2.ElementAt(factoryJ)()); + } + } + } + + public static IEnumerable> GetSpecialPairs(IEnumerable>> pairFactories) + { + foreach (var pairFactory in pairFactories) + { + yield return pairFactory(); + } + } + + public static IEnumerable> GetEqualPairs(IEnumerable> valueFactories) + { + foreach (var factory in valueFactories) + { + yield return Tuple.Create(factory(), factory()); + } + } + + public static IEnumerable> GetUnequalPairs(IEnumerable> valueFactories) + { + var set = valueFactories; + for (var factoryI = 0; factoryI < set.Count(); factoryI++) + { + for (var factoryJ = 0; factoryJ < set.Count(); factoryJ++) + { + if (factoryI != factoryJ) + { + yield return Tuple.Create(set.ElementAt(factoryI)(), set.ElementAt(factoryJ)()); + } + } + } + } + + public static IEnumerable GetImplementations() + { + var types = + Assembly.GetExecutingAssembly().GetTypes() + .Where(p => + typeof(T) + .IsAssignableFrom(p) + && !p.IsAbstract); + + return types.Select( + t => (T)t.GetConstructor(new Type[] { }).Invoke(null)); + } + + public static ValidationContext GetFakeValidationContext() + { + return ValidationContextStub.Create(new PackageIdentity("testPackage", new NuGetVersion(1, 0, 0))); + } + + internal static void AddPackageToMockServer(MockServerHttpClientHandler clientHandler, PackageIdentity packageIdentity, string filePath) + { + string packageId = packageIdentity.Id.ToLowerInvariant(); + string packageVersion = packageIdentity.Version.ToNormalizedString().ToLowerInvariant(); + + clientHandler.SetAction($"/packages/{packageId}/{packageVersion}/{packageId}.{packageVersion}.nupkg", request => + { + byte[] bytes = File.ReadAllBytes(filePath); + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(bytes) + }); + }); + } + + internal static void AddCatalogLeafToMockServer(MockServerHttpClientHandler clientHandler, Uri uri, CatalogLeaf leaf) + { + string relativeUrl = uri.IsAbsoluteUri ? uri.AbsolutePath : uri.ToString(); + + clientHandler.SetAction(relativeUrl, request => + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonConvert.SerializeObject(leaf)) + }); + }); + } + } +} \ No newline at end of file diff --git a/tests/NgTests/Validation/ValidatorTests.cs b/tests/NgTests/Validation/ValidatorTests.cs new file mode 100644 index 000000000..ea9ef642b --- /dev/null +++ b/tests/NgTests/Validation/ValidatorTests.cs @@ -0,0 +1,152 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Services.Metadata.Catalog.Monitoring; +using Xunit; + +namespace NgTests.Validation +{ + public class ValidatorTests + { + [Fact] + public async Task Run_ReturnsPass() + { + // Arrange + Func shouldRun = () => ShouldRunTestResult.Yes; + Action runInternal = () => { }; + + var validator = new TestableValidator(shouldRun, runInternal); + + var context = CreateContext(); + + // Act + var result = await validator.ValidateAsync(context); + + // Assert + Assert.Same(validator, result.Validator); + Assert.Equal(TestResult.Pass, result.Result); + Assert.Null(result.Exception); + } + + [Fact] + public async Task Run_ReturnsSkip() + { + // Arrange + Func shouldRun = () => ShouldRunTestResult.No; + Action runInternal = () => { }; + + var validator = new TestableValidator(shouldRun, runInternal); + + var context = CreateContext(); + + // Act + var result = await validator.ValidateAsync(context); + + // Assert + Assert.Same(validator, result.Validator); + Assert.Equal(TestResult.Skip, result.Result); + Assert.Null(result.Exception); + } + + [Fact] + public async Task Run_ReturnsPending() + { + // Arrange + Func shouldRun = () => ShouldRunTestResult.RetryLater; + Action runInternal = () => { }; + + var validator = new TestableValidator(shouldRun, runInternal); + + var context = CreateContext(); + + // Act + var result = await validator.ValidateAsync(context); + + // Assert + Assert.Same(validator, result.Validator); + Assert.Equal(TestResult.Pending, result.Result); + Assert.Null(result.Exception); + } + + [Fact] + public async Task Run_ReturnsFail() + { + // Arrange + var exception = new Exception(); + + Func shouldRun = () => ShouldRunTestResult.Yes; + Action runInternal = () => { throw exception; }; + + var validator = new TestableValidator(shouldRun, runInternal); + + var context = CreateContext(); + + // Act + var result = await validator.ValidateAsync(context); + + // Assert + Assert.Same(validator, result.Validator); + Assert.Equal(TestResult.Fail, result.Result); + Assert.Same(exception, result.Exception); + } + + private static ValidationContext CreateContext() + { + return ValidationContextStub.Create(); + } + } + + public class TestableValidator : Validator + { + private static readonly ILogger _logger; + private static readonly ValidatorConfiguration _validatorConfiguration; + + static TestableValidator() + { + var sourceRepository = new Mock(); + var metadataResource = new Mock(); + + metadataResource.Setup(x => x.GetAsync(It.IsAny())) + .ReturnsAsync(PackageTimestampMetadata.CreateForExistingPackage(DateTime.Now, DateTime.Now)); + + sourceRepository.Setup(x => x.GetResource()) + .Returns(metadataResource.Object); + + var feedToSource = new Mock>(); + + feedToSource.Setup(x => x[It.IsAny()]).Returns(sourceRepository.Object); + + _validatorConfiguration = new ValidatorConfiguration(packageBaseAddress: "a", requireRepositorySignature: true); + _logger = Mock.Of>(); + } + + public TestableValidator(Func shouldRun, Action runInternal) + : base(_validatorConfiguration, _logger) + { + _shouldRun = shouldRun; + _runInternal = runInternal; + } + + protected override Task ShouldRunAsync(ValidationContext context) + { + return Task.FromResult(_shouldRun()); + } + + protected override Task RunInternalAsync(ValidationContext context) + { + _runInternal(); + + return Task.FromResult(0); + } + + private Func _shouldRun; + private Action _runInternal; + } +} \ No newline at end of file diff --git a/tests/NuGet.Jobs.Catalog2Registration.Tests/App.config b/tests/NuGet.Jobs.Catalog2Registration.Tests/App.config new file mode 100644 index 000000000..2a2d44990 --- /dev/null +++ b/tests/NuGet.Jobs.Catalog2Registration.Tests/App.config @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/NuGet.Jobs.Catalog2Registration.Tests/Catalog2RegistrationCommandFacts.cs b/tests/NuGet.Jobs.Catalog2Registration.Tests/Catalog2RegistrationCommandFacts.cs new file mode 100644 index 000000000..8f55f1583 --- /dev/null +++ b/tests/NuGet.Jobs.Catalog2Registration.Tests/Catalog2RegistrationCommandFacts.cs @@ -0,0 +1,159 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage.Blob; +using Moq; +using Moq.Protected; +using NuGet.Services; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Services.V3; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class Catalog2RegistrationCommandFacts + { + public class TheExecuteAsyncMethod : Facts + { + public TheExecuteAsyncMethod(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task LoadsCursorsAndExecutesCollector() + { + await Target.ExecuteAsync(); + + HttpMessageHandler.Verify( + x => x.OnSendAsync(It.IsAny(), It.IsAny()), + Times.Never); + CloudBlobClient.Verify(x => x.GetContainerReference(It.IsAny()), Times.Never); + Storage.Protected().Verify( + "OnLoadAsync", + Times.Once(), + ItExpr.IsAny(), + ItExpr.IsAny()); + Storage.Protected().Verify( + "OnLoadAsync", + Times.Once(), + new Uri("https://example/azs/cursor.json"), + ItExpr.IsAny()); + Collector.Verify( + x => x.RunAsync(It.IsAny(), It.Is(c => c.Value == DateTime.MaxValue.ToUniversalTime()), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task CreatesContainersIfConfigured() + { + Config.CreateContainers = true; + + await Target.ExecuteAsync(); + + CloudBlobClient.Verify(x => x.GetContainerReference(It.IsAny()), Times.Exactly(3)); + CloudBlobClient.Verify(x => x.GetContainerReference(Config.LegacyStorageContainer), Times.Once); + CloudBlobClient.Verify(x => x.GetContainerReference(Config.GzippedStorageContainer), Times.Once); + CloudBlobClient.Verify(x => x.GetContainerReference(Config.SemVer2StorageContainer), Times.Once); + CloudBlobContainer.Verify(x => x.CreateIfNotExistAsync(), Times.Exactly(3)); + CloudBlobContainer.Verify( + x => x.SetPermissionsAsync(It.Is(p => p.PublicAccess == BlobContainerPublicAccessType.Blob)), + Times.Exactly(3)); + } + + [Fact] + public async Task LoadsDependencyCursorsIfConfigured() + { + Config.DependencyCursorUrls = new List() + { + "https://example/fc-1/cursor.json", + "https://example/fc-2/cursor.json", + }; + HttpMessageHandler + .Setup(x => x.OnSendAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent($"{{\"value\": \"{DateTimeOffset.UtcNow.ToString("O")}\"}}"), + }); + + await Target.ExecuteAsync(); + + HttpMessageHandler.Verify( + x => x.OnSendAsync(It.IsAny(), It.IsAny()), + Times.Exactly(2)); + HttpMessageHandler.Verify( + x => x.OnSendAsync( + It.Is(r => r.RequestUri.AbsoluteUri == "https://example/fc-1/cursor.json"), + It.IsAny()), + Times.Once); + HttpMessageHandler.Verify( + x => x.OnSendAsync( + It.Is(r => r.RequestUri.AbsoluteUri == "https://example/fc-2/cursor.json"), + It.IsAny()), + Times.Once); + Storage.Protected().Verify( + "OnLoadAsync", + Times.Once(), + new Uri("https://example/azs/cursor.json"), + ItExpr.IsAny()); + Collector.Verify( + x => x.RunAsync(It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + Collector = new Mock(); + CloudBlobClient = new Mock(); + StorageFactory = new Mock(); + HttpMessageHandler = new Mock { CallBase = true }; + Options = new Mock>(); + Logger = output.GetLogger(); + + CloudBlobContainer = new Mock(); + CloudBlobClient.Setup(x => x.GetContainerReference(It.IsAny())).Returns(() => CloudBlobContainer.Object); + Storage = new Mock(new Uri("https://example/azs")); + StorageFactory.Setup(x => x.Create(It.IsAny())).Returns(() => Storage.Object); + Config = new Catalog2RegistrationConfiguration + { + StorageConnectionString = "UseDevelopmentStorage=true", + LegacyStorageContainer = "reg", + GzippedStorageContainer = "reg-gz", + SemVer2StorageContainer = "reg-gz-semver2", + }; + Options.Setup(x => x.Value).Returns(() => Config); + + Target = new Catalog2RegistrationCommand( + Collector.Object, + CloudBlobClient.Object, + StorageFactory.Object, + () => HttpMessageHandler.Object, + Options.Object, + Logger); + } + + public Mock Collector { get; } + public Mock CloudBlobClient { get; } + public Mock StorageFactory { get; } + public Mock HttpMessageHandler { get; } + public Mock> Options { get; } + public RecordingLogger Logger { get; } + public Mock Storage { get; } + public Mock CloudBlobContainer { get; } + public Catalog2RegistrationConfiguration Config { get; } + public Catalog2RegistrationCommand Target { get; } + } + } +} diff --git a/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveMergerFacts.FullEnumeration.cs b/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveMergerFacts.FullEnumeration.cs new file mode 100644 index 000000000..c093bfa6d --- /dev/null +++ b/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveMergerFacts.FullEnumeration.cs @@ -0,0 +1,317 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using NuGet.Services.Metadata.Catalog.Helpers; +using NuGet.Services.V3.Support; +using NuGet.Versioning; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Jobs.Catalog2Registration +{ + public partial class HiveMergerFacts + { + public class FullEnumeration + { + private readonly ITestOutputHelper _output; + + public FullEnumeration(ITestOutputHelper output) + { + _output = output; + } + + [Theory] + [InlineData(1, 2, 4, 1, 7)] // Use page size 1 to try some extreme prepending cases. + [InlineData(2, 2, 6, 1, 7)] // Use page size 2 so we can easily have up to 3 pages. + [InlineData(3, 2, 5, 1, 6)] // Use page size 3 to allow insertions without bounds changes. + public async Task Execute(int maxLeavesPerPage, int minExisting, int maxExisting, int minUpdated, int maxUpdated) + { + var config = new Catalog2RegistrationConfiguration(); + var options = new SimpleOptions(config); + var logger = new NullLogger(); + var target = new HiveMerger(options, logger); + + config.MaxLeavesPerPage = maxLeavesPerPage; + + var allExisting = Enumerable.Range(minExisting, (maxExisting - minExisting) + 1).Select(m => new NuGetVersion(m, 0, 0)).ToList(); + var allUpdated = Enumerable.Range(minUpdated, (maxUpdated - minUpdated) + 1).Select(m => new NuGetVersion(m, 0, 0)).ToList(); + var versionToNormalized = allExisting.Concat(allUpdated).Distinct().ToDictionary(v => v, v => v.ToNormalizedString()); + + // Enumerate all of the updated version sets, which is up to 9 versions and for each set every + // combination of either PackageDelete or PackageDetails for each version. + var deleteOrNotDelete = new[] { true, false }; + var updatedCases = IterTools + .SubsetsOf(allUpdated) + .SelectMany(vs => IterTools.CombinationsOfTwo(vs.ToList(), deleteOrNotDelete)) + .Select(ts => ts.Select(t => new VersionAction(t.Item1, t.Item2)).OrderBy(v => v.Version).ToList()) + .ToList(); + + // Enumerate all of the updated version sets, which is up to 7 versions. + var existingCases = IterTools + .SubsetsOf(allExisting) + .Select(vs => vs.OrderBy(v => v).ToList()) + .ToList(); + + // Build all of the test cases which is every pairing of updated and existing cases. + var testCases = new ConcurrentBag(); + foreach (var existing in existingCases) + { + foreach (var updated in updatedCases) + { + testCases.Add(new TestCase(existing, updated)); + } + } + + var completed = 0; + var total = testCases.Count; + var outputLock = new object(); + await ParallelAsync + .Repeat(async () => + { + await Task.Yield(); + while (testCases.TryTake(out var testCase)) + { + try + { + await ExecuteTestCaseAsync(config, target, versionToNormalized, testCase); + + var thisCompleted = Interlocked.Increment(ref completed); + if (thisCompleted % 20000 == 0) + { + lock (outputLock) + { + _output.WriteLine($"Progress: {1.0 * thisCompleted / total:P}"); + } + } + } + catch (Exception ex) + { + lock (outputLock) + { + _output.WriteLine(string.Empty); + _output.WriteLine("Test Case Failure"); + _output.WriteLine(new string('=', 50)); + _output.WriteLine(ex.Message); + _output.WriteLine(string.Empty); + _output.WriteLine("Existing: " + string.Join(", ", testCase.Existing)); + _output.WriteLine("Changes: " + string.Join(", ", testCase.Updated.Select(t => $"{(t.IsDelete ? '-' : '+')}{t.Version}"))); + _output.WriteLine(new string('=', 50)); + _output.WriteLine(string.Empty); + return; + } + } + } + }, + degreeOfParallelism: 8); + + _output.WriteLine($"Progress: {1.0 * completed / total:P}"); + _output.WriteLine($"Total test cases: {total}"); + _output.WriteLine($"Completed test cases: {completed}"); + Assert.Equal(total, completed); + } + + private async Task ExecuteTestCaseAsync( + Catalog2RegistrationConfiguration config, + HiveMerger target, + Dictionary versionToNormalized, + TestCase testCase) + { + // ARRANGE + var existing = testCase.Existing; + var allUpdated = testCase.Updated; + + // Determine the sets of expected modified versions, deleted versions, and resulting versions. + var updatedGrouped = allUpdated.ToLookup(x => x.IsDelete); + var possibleDeletes = updatedGrouped[true].Select(x => x.Version); + var modified = updatedGrouped[false].Select(x => x.Version).OrderBy(v => v).ToList(); + var deleted = existing.Intersect(possibleDeletes).OrderBy(v => v).ToList(); + var unchanged = existing.Except(allUpdated.Select(x => x.Version)).OrderBy(v => v).ToList(); + var expectedRemaining = modified.Concat(unchanged).OrderBy(v => v).ToList(); + + // Build the input data structures. + var sortedCatalog = MakeSortedCatalog(allUpdated); + var indexInfo = MakeIndexInfo(existing, config.MaxLeavesPerPage, versionToNormalized); + + // Determine which pages will definitely not be affected. + var unaffectedPages = new List(); + if (existing.Any()) + { + var minVersion = allUpdated.Min(x => x.Version); + unaffectedPages = indexInfo + .Items + .Take(indexInfo.Items.Count - 1) + .Where(x => x.Upper < minVersion) + .ToList(); + } + + // ACT + var result = await target.MergeAsync(indexInfo, sortedCatalog); + + // ASSERT + + // Verify the definitely unaffected pages are in still in the index and not loaded. + foreach (var page in unaffectedPages) + { + Assert.Contains(page, indexInfo.Items); + Assert.False(page.IsPageFetched, "A page before the lowest version input version must not be fetched."); + } + + // Verify the modified version set. + Assert.Equal(modified, result.ModifiedLeaves.Select(x => x.Version).OrderBy(v => v).ToList()); + + // Verify the deleted version set. + Assert.Equal(deleted, result.DeletedLeaves.Select(x => x.Version).OrderBy(v => v).ToList()); + + // Verify the resulting set of versions. + var actualRemaining = await GetVersionsAsync(indexInfo); + Assert.Equal(expectedRemaining, actualRemaining); + + // Verify all but the last page are full. + foreach (var pageInfo in indexInfo.Items.AsEnumerable().Reverse().Skip(1)) + { + Assert.True( + pageInfo.Count == config.MaxLeavesPerPage, + "All but the last page must have the maximum number of leaf items per page."); + } + + // Verify the last page is not too full. + if (indexInfo.Items.Count > 0) + { + Assert.True( + indexInfo.Items.Last().Count <= config.MaxLeavesPerPage, + "The last page must have less than or equal to the maximum number of leaf items per page."); + } + + // Verify the page bounds are in ascending order. + var bounds = indexInfo.Items.SelectMany(GetUniqueBounds).ToList(); + for (var i = 1; i < bounds.Count; i++) + { + Assert.True(bounds[i - 1] < bounds[i], "Each page bound must be less than the next."); + } + + // Verify the modified pages are in the index. + foreach (var pageInfo in result.ModifiedPages) + { + Assert.True(indexInfo.Items.Contains(pageInfo), "A modified page must be in the index."); + } + + // Verify the leaf item infos match the leaf items. + Assert.True( + indexInfo.Items.Count == indexInfo.Index.Items.Count, + "The number of page infos must match the number of page items."); + for (var pageIndex = 0; pageIndex < indexInfo.Items.Count; pageIndex++) + { + var pageInfo = indexInfo.Items[pageIndex]; + var leafInfos = await pageInfo.GetLeafInfosAsync(); + var page = await pageInfo.GetPageAsync(); + Assert.True( + page.Items.Count == leafInfos.Count, + "The number of leaf items must match the number of leaf item infos."); + + for (var leafIndex = 0; leafIndex < page.Items.Count; leafIndex++) + { + var leafInfoVersion = leafInfos[leafIndex].Version; + var leafItemVersion = NuGetVersion.Parse(page.Items[leafIndex].CatalogEntry.Version); + Assert.True( + leafInfoVersion == leafItemVersion, + "The list of leaf info versions must match the leaf item versions."); + } + } + + // Verify the modified leaves and deleted leafs are disjoint sets. + var modifiedVersions = new HashSet(result.ModifiedLeaves.Select(x => x.Version)); + var deletedVersions = new HashSet(result.DeletedLeaves.Select(x => x.Version)); + Assert.True( + !modifiedVersions.Overlaps(deletedVersions), + "The deleted leaves and modified leaves must be disjoint sets."); + + // Verify the modified leaves are visible in an inlined page or a downloaded page. + foreach (var leafInfo in result.ModifiedLeaves) + { + foreach (var pageInfo in indexInfo.Items) + { + if (leafInfo.Version < pageInfo.Lower || leafInfo.Version > pageInfo.Upper) + { + continue; + } + + Assert.True( + pageInfo.IsInlined || pageInfo.IsPageFetched, + "A page bounding a modified leaf must either be inlined or the page must be fetched."); + + var pageLeafInfos = await pageInfo.GetLeafInfosAsync(); + + Assert.True( + pageLeafInfos.Any(x => x.Version == leafInfo.Version), + "A modified leaf must be in the page bounding its version."); + } + } + + // Verify the deleted leaves are not visible in any of the inlined or downloaded pages. + foreach (var leafInfo in result.DeletedLeaves) + { + foreach (var pageInfo in indexInfo.Items) + { + var pageLeafInfos = await pageInfo.GetLeafInfosAsync(); + + Assert.True( + !pageLeafInfos.Any(x => x.Version == leafInfo.Version), + "A deleted leaf must not be in any page."); + } + } + } + + private static IEnumerable GetUniqueBounds(PageInfo x) + { + yield return x.Lower; + + if (x.Lower != x.Upper) + { + yield return x.Upper; + } + else + { + Assert.True( + x.Count == 1, + "If the upper bound and the lower bound are the same, the leaf item count must be one."); + } + } + + private class SimpleOptions : IOptionsSnapshot where T : class, new() + { + public SimpleOptions(T value) + { + Value = value; + } + + public T Value { get; } + public T Get(string name) => throw new NotImplementedException(); + } + + private class NullLogger : ILogger + { + public IDisposable BeginScope(TState state) + { + return null; + } + + public bool IsEnabled(LogLevel logLevel) + { + return false; + } + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + } + } + } + } +} diff --git a/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveMergerFacts.MergeAsync.cs b/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveMergerFacts.MergeAsync.cs new file mode 100644 index 000000000..f554a58fc --- /dev/null +++ b/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveMergerFacts.MergeAsync.cs @@ -0,0 +1,455 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NuGet.Versioning; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Jobs.Catalog2Registration +{ + public partial class HiveMergerFacts + { + public class TheMergeAsyncMethod : Facts + { + public TheMergeAsyncMethod(ITestOutputHelper output) : base(output) + { + } + + [Theory] + [MemberData(nameof(Versions))] + public async Task AddSingleFirstVersion(string version) + { + var indexInfo = IndexInfo.New(); + var sortedCatalog = MakeSortedCatalog(Details(version)); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.True(indexInfo.Items[0].IsPageFetched); + + var pageInfo = Assert.Single(indexInfo.Items); + var page = await pageInfo.GetPageAsync(); + var leafInfo = Assert.Single(await pageInfo.GetLeafInfosAsync()); + var leafItem = Assert.Single(page.Items); + Assert.Same(leafItem, leafInfo.LeafItem); + Assert.Equal(version, leafItem.CatalogEntry.Version); + + Assert.Same(pageInfo, Assert.Single(result.ModifiedPages)); + Assert.Same(leafInfo, Assert.Single(result.ModifiedLeaves)); + Assert.Empty(result.DeletedLeaves); + } + + [Theory] + [MemberData(nameof(Versions))] + public async Task UpdateSingleVersion(string version) + { + var indexInfo = MakeIndexInfo("1.0.0", "2.0.0", version, "4.0.0", "5.0.0"); + var sortedCatalog = MakeSortedCatalog(Details(version)); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.True(indexInfo.Items[0].IsPageFetched); + Assert.False(indexInfo.Items[1].IsPageFetched); + + Assert.Equal(2, indexInfo.Items.Count); + var pageA = await indexInfo.Items[0].GetPageAsync(); + var pageB = await indexInfo.Items[1].GetPageAsync(); + Assert.Equal(3, pageA.Items.Count); + Assert.Equal("1.0.0", pageA.Items[0].CatalogEntry.Version); + Assert.Equal("2.0.0", pageA.Items[1].CatalogEntry.Version); + Assert.Equal(version, pageA.Items[2].CatalogEntry.Version); + Assert.Equal(2, pageB.Items.Count); + Assert.Equal("4.0.0", pageB.Items[0].CatalogEntry.Version); + Assert.Equal("5.0.0", pageB.Items[1].CatalogEntry.Version); + + var pageInfo = Assert.Single(result.ModifiedPages); + Assert.Same(indexInfo.Items[0], pageInfo); + var leafInfo = (await pageInfo.GetLeafInfosAsync())[2]; + Assert.Same(leafInfo, Assert.Single(result.ModifiedLeaves)); + Assert.Empty(result.DeletedLeaves); + } + + [Theory] + [MemberData(nameof(Versions))] + public async Task RemoveSingleExisting(string version) + { + var indexInfo = MakeIndexInfo("1.0.0", "2.0.0", version, "4.0.0", "5.0.0"); + var sortedCatalog = MakeSortedCatalog(Delete(version)); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.True(indexInfo.Items[0].IsPageFetched); + Assert.True(indexInfo.Items[1].IsPageFetched); + + Assert.Equal(2, indexInfo.Items.Count); + var pageA = await indexInfo.Items[0].GetPageAsync(); + var pageB = await indexInfo.Items[1].GetPageAsync(); + Assert.Equal(3, pageA.Items.Count); + Assert.Equal("1.0.0", pageA.Items[0].CatalogEntry.Version); + Assert.Equal("2.0.0", pageA.Items[1].CatalogEntry.Version); + Assert.Equal("4.0.0", pageA.Items[2].CatalogEntry.Version); + Assert.Equal("5.0.0", Assert.Single(pageB.Items).CatalogEntry.Version); + + Assert.Contains(indexInfo.Items[0], result.ModifiedPages); + Assert.Contains(indexInfo.Items[1], result.ModifiedPages); + Assert.Empty(result.ModifiedLeaves); + Assert.Equal(version, Assert.Single(result.DeletedLeaves).LeafItem.CatalogEntry.Version); + } + + [Fact] + public async Task DeleteAgainstEmpty() + { + var indexInfo = IndexInfo.New(); + var sortedCatalog = MakeSortedCatalog(Delete("1.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.Empty(indexInfo.Items); + + Assert.Empty(result.ModifiedPages); + Assert.Empty(result.ModifiedLeaves); + Assert.Empty(result.DeletedLeaves); + } + + [Fact] + public async Task DeleteLast() + { + var indexInfo = MakeIndexInfo("1.0.0"); + var leafInfo = (await indexInfo.Items.First().GetLeafInfosAsync()).First(); + var sortedCatalog = MakeSortedCatalog(Delete("1.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.Empty(indexInfo.Items); + + Assert.Empty(result.ModifiedPages); + Assert.Empty(result.ModifiedLeaves); + Assert.Same(leafInfo, Assert.Single(result.DeletedLeaves)); + } + + [Fact] + public async Task DeleteLastThenAddOne() + { + var indexInfo = MakeIndexInfo("1.0.0"); + var leafInfoA = (await indexInfo.Items.First().GetLeafInfosAsync()).First(); + var sortedCatalog = MakeSortedCatalog(Delete("1.0.0"), Details("2.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.True(indexInfo.Items[0].IsPageFetched); + + var pageInfo = Assert.Single(indexInfo.Items); + var page = await pageInfo.GetPageAsync(); + var leafInfoB = Assert.Single(await pageInfo.GetLeafInfosAsync()); + var leafItem = Assert.Single(page.Items); + Assert.Same(leafItem, leafInfoB.LeafItem); + Assert.NotSame(leafInfoA, leafInfoB); + Assert.Equal("2.0.0", leafItem.CatalogEntry.Version); + + Assert.Same(pageInfo, Assert.Single(result.ModifiedPages)); + Assert.Same(leafInfoB, Assert.Single(result.ModifiedLeaves)); + Assert.Same(leafInfoA, Assert.Single(result.DeletedLeaves)); + } + + [Fact] + public async Task DeleteNonExistentThenAddOne() + { + var indexInfo = MakeIndexInfo("1.0.0"); + var sortedCatalog = MakeSortedCatalog(Delete("2.0.0"), Details("3.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.True(indexInfo.Items[0].IsPageFetched); + + var pageInfo = Assert.Single(indexInfo.Items); + var leafInfos = await pageInfo.GetLeafInfosAsync(); + Assert.Equal(2, leafInfos.Count); + Assert.Equal("1.0.0", leafInfos[0].LeafItem.CatalogEntry.Version); + Assert.Equal("3.0.0", leafInfos[1].LeafItem.CatalogEntry.Version); + + Assert.Same(pageInfo, Assert.Single(result.ModifiedPages)); + Assert.Same(leafInfos[1], Assert.Single(result.ModifiedLeaves)); + Assert.Empty(result.DeletedLeaves); + } + + [Fact] + public async Task RemoveLastVersionFromPage() + { + var indexInfo = MakeIndexInfo("1.0.0", "2.0.0", "3.0.0", "4.0.0"); + var sortedCatalog = MakeSortedCatalog(Delete("4.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.False(indexInfo.Items[0].IsPageFetched); + + var pageInfo = Assert.Single(indexInfo.Items); + var page = await pageInfo.GetPageAsync(); + Assert.Equal(3, page.Items.Count); + Assert.Equal("1.0.0", page.Items[0].CatalogEntry.Version); + Assert.Equal("2.0.0", page.Items[1].CatalogEntry.Version); + Assert.Equal("3.0.0", page.Items[2].CatalogEntry.Version); + + Assert.Empty(result.ModifiedPages); + Assert.Empty(result.ModifiedLeaves); + Assert.Equal("4.0.0", Assert.Single(result.DeletedLeaves).LeafItem.CatalogEntry.Version); + } + + [Fact] + public async Task RemoveVersionFromPageCausingLastPageToBeRemoved() + { + var indexInfo = MakeIndexInfo("1.0.0", "2.0.0", "3.0.0", "4.0.0"); + var sortedCatalog = MakeSortedCatalog(Delete("3.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.True(indexInfo.Items[0].IsPageFetched); + + var pageInfo = Assert.Single(indexInfo.Items); + Assert.Equal(new[] { "1.0.0", "2.0.0", "4.0.0" }, await GetVersionArrayAsync(indexInfo)); + + Assert.Same(pageInfo, Assert.Single(result.ModifiedPages)); + Assert.Empty(result.ModifiedLeaves); + Assert.Equal("3.0.0", Assert.Single(result.DeletedLeaves).LeafItem.CatalogEntry.Version); + } + + [Fact] + public async Task InsertInMiddlePage() + { + var indexInfo = MakeIndexInfo("1.0.0", "2.0.0", "3.0.0", "4.0.0", "6.0.0", "7.0.0", "8.0.0"); + var sortedCatalog = MakeSortedCatalog(Details("5.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.False(indexInfo.Items[0].IsPageFetched); + Assert.True(indexInfo.Items[1].IsPageFetched); + Assert.True(indexInfo.Items[2].IsPageFetched); + + Assert.Equal(3, indexInfo.Items.Count); + Assert.Equal( + new[] { "1.0.0", "2.0.0", "3.0.0", "4.0.0", "5.0.0", "6.0.0", "7.0.0", "8.0.0" }, + await GetVersionArrayAsync(indexInfo)); + + Assert.Equal(2, result.ModifiedPages.Count); + Assert.DoesNotContain(indexInfo.Items[0], result.ModifiedPages); + Assert.Contains(indexInfo.Items[1], result.ModifiedPages); + Assert.Contains(indexInfo.Items[2], result.ModifiedPages); + Assert.Equal("5.0.0", Assert.Single(result.ModifiedLeaves).LeafItem.CatalogEntry.Version); + Assert.Empty(result.DeletedLeaves); + } + + [Fact] + public async Task InsertInLastPage() + { + var indexInfo = MakeIndexInfo("1.0.0", "2.0.0", "3.0.0", "4.0.0", "5.0.0", "6.0.0", "8.0.0"); + var sortedCatalog = MakeSortedCatalog(Details("7.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.False(indexInfo.Items[0].IsPageFetched); + Assert.False(indexInfo.Items[1].IsPageFetched); + Assert.True(indexInfo.Items[2].IsPageFetched); + + Assert.Equal(3, indexInfo.Items.Count); + Assert.Equal( + new[] { "1.0.0", "2.0.0", "3.0.0", "4.0.0", "5.0.0", "6.0.0", "7.0.0", "8.0.0" }, + await GetVersionArrayAsync(indexInfo)); + + Assert.Single(result.ModifiedPages); + Assert.Contains(indexInfo.Items[2], result.ModifiedPages); + Assert.Equal("7.0.0", Assert.Single(result.ModifiedLeaves).LeafItem.CatalogEntry.Version); + Assert.Empty(result.DeletedLeaves); + } + + [Fact] + public async Task InsertLowestCreatingNewPage() + { + var indexInfo = MakeIndexInfo("2.0.0", "3.0.0", "4.0.0"); + var sortedCatalog = MakeSortedCatalog(Details("1.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.True(indexInfo.Items[0].IsPageFetched); + Assert.True(indexInfo.Items[1].IsPageFetched); + + Assert.Equal(2, indexInfo.Items.Count); + Assert.Equal(new[] { "1.0.0", "2.0.0", "3.0.0", "4.0.0" }, await GetVersionArrayAsync(indexInfo)); + + Assert.Equal(2, result.ModifiedPages.Count); + Assert.Contains(indexInfo.Items[0], result.ModifiedPages); + Assert.Contains(indexInfo.Items[1], result.ModifiedPages); + Assert.Equal("1.0.0", Assert.Single(result.ModifiedLeaves).LeafItem.CatalogEntry.Version); + Assert.Empty(result.DeletedLeaves); + } + + [Fact] + public async Task AppendLatestVersionCreatingNewPage() + { + var indexInfo = MakeIndexInfo("1.0.0", "2.0.0", "3.0.0"); + var sortedCatalog = MakeSortedCatalog(Details("4.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.False(indexInfo.Items[0].IsPageFetched); + Assert.True(indexInfo.Items[1].IsPageFetched); + + Assert.Equal(2, indexInfo.Items.Count); + Assert.Equal(new[] { "1.0.0", "2.0.0", "3.0.0", "4.0.0" }, await GetVersionArrayAsync(indexInfo)); + + Assert.Equal(indexInfo.Items[1], Assert.Single(result.ModifiedPages)); + Assert.Equal("4.0.0", Assert.Single(result.ModifiedLeaves).LeafItem.CatalogEntry.Version); + Assert.Empty(result.DeletedLeaves); + } + + [Fact] + public async Task InterleaveVersions() + { + var indexInfo = MakeIndexInfo("2.0.0", "4.0.0", "6.0.0", "8.0.0", "10.0.0"); + var sortedCatalog = MakeSortedCatalog( + Details("1.0.0"), + Details("3.0.0"), + Details("5.0.0"), + Details("7.0.0"), + Details("9.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.True(indexInfo.Items[0].IsPageFetched); + Assert.True(indexInfo.Items[1].IsPageFetched); + Assert.True(indexInfo.Items[2].IsPageFetched); + Assert.True(indexInfo.Items[3].IsPageFetched); + + Assert.Equal(4, indexInfo.Items.Count); + Assert.Equal( + new[] { "1.0.0", "2.0.0", "3.0.0", "4.0.0", "5.0.0", "6.0.0", "7.0.0", "8.0.0", "9.0.0", "10.0.0" }, + await GetVersionArrayAsync(indexInfo)); + + Assert.Equal(4, result.ModifiedPages.Count); + Assert.Equal(new[] { "1.0.0", "3.0.0", "5.0.0", "7.0.0", "9.0.0" }, GetVersionArray(result.ModifiedLeaves)); + Assert.Empty(result.DeletedLeaves); + } + + [Fact] + public async Task DeleteAllVersions() + { + var indexInfo = MakeIndexInfo("1.0.0", "2.0.0", "3.0.0", "4.0.0"); + var sortedCatalog = MakeSortedCatalog(Delete("1.0.0"), Delete("2.0.0"), Delete("3.0.0"), Delete("4.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.Empty(indexInfo.Items); + Assert.Empty(await GetVersionArrayAsync(indexInfo)); + + Assert.Empty(result.ModifiedPages); + Assert.Empty(result.ModifiedLeaves); + Assert.Equal(new[] { "1.0.0", "2.0.0", "3.0.0", "4.0.0" }, GetVersionArray(result.DeletedLeaves)); + } + + [Fact] + public async Task AddManyVersions() + { + var indexInfo = IndexInfo.New(); + var sortedCatalog = MakeSortedCatalog(Details("1.0.0"), Details("2.0.0"), Details("3.0.0"), Details("4.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.Equal(2, indexInfo.Items.Count); + Assert.Equal(new[] { "1.0.0", "2.0.0", "3.0.0", "4.0.0" }, await GetVersionArrayAsync(indexInfo)); + + Assert.Equal(2, result.ModifiedPages.Count); + Assert.Equal(new[] { "1.0.0", "2.0.0", "3.0.0", "4.0.0" }, GetVersionArray(result.ModifiedLeaves)); + Assert.Empty(result.DeletedLeaves); + } + + [Theory] + [InlineData("2.0.0", "1.0.0")] + [InlineData("4.0.0", "1.0.0")] + [InlineData("6.0.0", "1.0.0")] + [InlineData("2.0.0", "3.0.0")] + [InlineData("4.0.0", "3.0.0")] + [InlineData("6.0.0", "3.0.0")] + [InlineData("2.0.0", "5.0.0")] + [InlineData("4.0.0", "5.0.0")] + [InlineData("6.0.0", "5.0.0")] + [InlineData("2.0.0", "7.0.0")] + [InlineData("4.0.0", "7.0.0")] + [InlineData("6.0.0", "7.0.0")] + public async Task AddAndRemoveFromNonLastPage(string deleted, string added) + { + var existing = new[] { "2.0.0", "4.0.0", "6.0.0", "8.0.0" }; + var expected = existing + .Except(new[] { deleted }) + .Concat(new[] { added }) + .OrderBy(x => NuGetVersion.Parse(x)) + .ToArray(); + var actions = new[] { Delete(deleted), Details(added) } + .OrderBy(x => x.Version) + .ToList(); + var indexInfo = MakeIndexInfo(existing); + var sortedCatalog = MakeSortedCatalog(actions); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.True(indexInfo.Items[0].IsPageFetched); + Assert.False(indexInfo.Items[1].IsPageFetched); + + Assert.Equal(2, indexInfo.Items.Count); + Assert.Equal(expected, await GetVersionArrayAsync(indexInfo)); + + Assert.Same(indexInfo.Items[0], Assert.Single(result.ModifiedPages)); + Assert.Equal(new[] { added }, GetVersionArray(result.ModifiedLeaves)); + Assert.Equal(new[] { deleted }, GetVersionArray(result.DeletedLeaves)); + } + + [Fact] + public async Task RemoveMiddlePage() + { + var indexInfo = MakeIndexInfo("1.0.0", "2.0.0", "3.0.0", "4.0.0", "5.0.0", "6.0.0", "7.0.0"); + var sortedCatalog = MakeSortedCatalog(Delete("4.0.0"), Delete("5.0.0"), Delete("6.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.False(indexInfo.Items[0].IsPageFetched); + Assert.False(indexInfo.Items[1].IsPageFetched); + + Assert.Equal(2, indexInfo.Items.Count); + Assert.Equal(new[] { "1.0.0", "2.0.0", "3.0.0", "7.0.0" }, await GetVersionArrayAsync(indexInfo)); + + Assert.Empty(result.ModifiedPages); + Assert.Empty(result.ModifiedLeaves); + Assert.Equal(new[] { "4.0.0", "5.0.0", "6.0.0" }, GetVersionArray(result.DeletedLeaves)); + } + + [Fact] + public async Task LoadsPageToFindMiss() + { + var indexInfo = MakeIndexInfo("1.0.0", "2.0.0", "4.0.0", "5.0.0"); + var sortedCatalog = MakeSortedCatalog(Delete("3.0.0")); + + var result = await Target.MergeAsync(indexInfo, sortedCatalog); + + Assert.True(indexInfo.Items[0].IsPageFetched); + Assert.False(indexInfo.Items[1].IsPageFetched); + + Assert.Equal(2, indexInfo.Items.Count); + Assert.Equal(new[] { "1.0.0", "2.0.0", "4.0.0", "5.0.0" }, await GetVersionArrayAsync(indexInfo)); + + Assert.Empty(result.ModifiedPages); + Assert.Empty(result.ModifiedLeaves); + Assert.Empty(result.DeletedLeaves); + } + + public static IEnumerable Versions => new[] + { + new object[] { "3.0.0" }, + new object[] { "3.1.0" }, + new object[] { "3.0.1" }, + new object[] { "3.0.0.1" }, + new object[] { "3.0.0-beta" }, + new object[] { "3.0.0-BETA" }, + new object[] { "3.0.0-beta.1" }, + }; + } + } +} diff --git a/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveMergerFacts.Support.cs b/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveMergerFacts.Support.cs new file mode 100644 index 000000000..94e162e40 --- /dev/null +++ b/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveMergerFacts.Support.cs @@ -0,0 +1,183 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Packaging.Core; +using NuGet.Protocol.Registration; +using NuGet.Services; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; +using Xunit.Abstractions; + +namespace NuGet.Jobs.Catalog2Registration +{ + public partial class HiveMergerFacts + { + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + Options = new Mock>(); + Logger = output.GetLogger(); + + Config = new Catalog2RegistrationConfiguration + { + MaxLeavesPerPage = 3, + }; + Options.Setup(x => x.Value).Returns(() => Config); + + Target = new HiveMerger( + Options.Object, + Logger); + } + + public Mock> Options { get; } + public RecordingLogger Logger { get; } + public Catalog2RegistrationConfiguration Config { get; } + public HiveMerger Target { get; } + } + + private class VersionAction + { + public VersionAction(NuGetVersion version, bool isDelete) + { + Version = version; + IsDelete = isDelete; + } + + public NuGetVersion Version { get; } + public bool IsDelete { get; } + } + + private class TestCase + { + public TestCase(List existing, List updated) + { + Existing = existing; + Updated = updated; + } + + public List Existing { get; } + public List Updated { get; } + } + + private static VersionAction Details(string version) + { + return new VersionAction(NuGetVersion.Parse(version), isDelete: false); + } + + private static VersionAction Delete(string version) + { + return new VersionAction(NuGetVersion.Parse(version), isDelete: true); + } + + private static string[] GetVersionArray(HashSet leafInfos) + { + return leafInfos + .OrderBy(x => x.Version) + .Select(x => x.LeafItem.CatalogEntry.Version) + .ToArray(); + } + + private static async Task GetVersionArrayAsync(IndexInfo indexInfo) + { + var versions = await GetVersionsAsync(indexInfo); + return versions.Select(x => x.ToNormalizedString()).ToArray(); + } + + private static async Task> GetVersionsAsync(IndexInfo indexInfo) + { + var versions = new List(); + foreach (var pageInfo in indexInfo.Items) + { + var leafInfos = await pageInfo.GetLeafInfosAsync(); + foreach (var leafInfo in leafInfos) + { + versions.Add(leafInfo.Version); + } + } + + return versions; + } + + private static List MakeSortedCatalog(params VersionAction[] versions) + { + return MakeSortedCatalog((ICollection)versions); + } + + private static List MakeSortedCatalog(ICollection sortedVersionActions) + { + var output = new List(); + foreach (var versionAction in sortedVersionActions) + { + var item = new CatalogCommitItem( + uri: null, + commitId: null, + commitTimeStamp: DateTime.MinValue, + types: null, + typeUris: new[] { versionAction.IsDelete ? Schema.DataTypes.PackageDelete : Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", versionAction.Version)); + + output.Add(item); + } + + return output; + } + + private static IndexInfo MakeIndexInfo(params string[] versions) + { + var sortedVersions = versions.Select(x => NuGetVersion.Parse(x)).ToList(); + var versionToNormalized = sortedVersions.ToDictionary(x => x, x => x.ToNormalizedString()); + + return MakeIndexInfo(sortedVersions, maxLeavesPerPage: 3, versionToNormalized: versionToNormalized); + } + + private static IndexInfo MakeIndexInfo( + List sortedVersions, + int maxLeavesPerPage, + Dictionary versionToNormalized) + { + var index = new RegistrationIndex + { + Items = new List(), + }; + + // Populate the pages. + RegistrationPage currentPage = null; + for (var i = 0; i < sortedVersions.Count; i++) + { + if (i % maxLeavesPerPage == 0) + { + currentPage = new RegistrationPage + { + Items = new List(), + }; + index.Items.Add(currentPage); + } + + currentPage.Items.Add(new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Version = versionToNormalized[sortedVersions[i]], + }, + }); + } + + // Update the bounds. + foreach (var page in index.Items) + { + page.Count = page.Items.Count; + page.Lower = page.Items.First().CatalogEntry.Version; + page.Upper = page.Items.Last().CatalogEntry.Version; + } + + return IndexInfo.Existing(storage: null, hive: HiveType.SemVer2, index: index); + } + } +} diff --git a/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveStorageFacts.cs b/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveStorageFacts.cs new file mode 100644 index 000000000..24bad8324 --- /dev/null +++ b/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveStorageFacts.cs @@ -0,0 +1,677 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Moq; +using Newtonsoft.Json; +using NuGet.Protocol; +using NuGet.Protocol.Catalog; +using NuGet.Protocol.Registration; +using NuGet.Services; +using NuGet.Versioning; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class HiveStorageFacts + { + public class ReadIndexOrNullAsync : Facts + { + public ReadIndexOrNullAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task ReturnsNullFor404() + { + LegacyBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .Throws(new StorageException(new RequestResult { HttpStatusCode = (int)HttpStatusCode.NotFound }, "Missing.", inner: null)); + + var index = await Target.ReadIndexOrNullAsync(HiveType.Legacy, "NuGet.Versioning"); + + Assert.Null(index); + } + + [Fact] + public async Task DeserializesGzipped() + { + GzippedBlob.Object.Properties.ContentEncoding = "gzip"; + var commitId = "6c6330d8-6d49-4c51-9a82-398a04f9e448"; + GzippedStream = SerializeToStream(new { commitId }, gzipped: true); + + var index = await Target.ReadIndexOrNullAsync(HiveType.Gzipped, "NuGet.Versioning"); + + Assert.Equal(commitId, index.CommitId); + } + + [Fact] + public async Task ProvidesUnencodedStringToStorage() + { + var index = await Target.ReadIndexOrNullAsync(HiveType.Legacy, "测试更新包"); + + LegacyContainer.Verify(x => x.GetBlobReference("测试更新包/index.json"), Times.Once); + } + + [Theory] + [MemberData(nameof(AllHivesTestData))] + public async Task DeserializesAllHiveTypes(HiveType hive) + { + var index = await Target.ReadIndexOrNullAsync(hive, "NuGet.Versioning"); + + Assert.NotNull(index); + CloudBlobClient.Verify(x => x.GetContainerReference(It.IsAny()), Times.Once); + CloudBlobClient.Verify(x => x.GetContainerReference(GetContainerName(hive)), Times.Once); + var container = GetContainer(hive); + container.Verify(x => x.GetBlobReference(It.IsAny()), Times.Once); + container.Verify(x => x.GetBlobReference("nuget.versioning/index.json"), Times.Once); + var blob = GetBlob(hive); + blob.Verify(x => x.OpenReadAsync(It.IsAny()), Times.Once); + blob.Verify(x => x.OpenReadAsync(It.Is(a => a.IfMatchETag == null && a.IfNoneMatchETag == null)), Times.Once); + Assert.Equal(blob.Object.Uri.AbsoluteUri, index.Url); + } + } + + public class ReadPageAsync : Facts + { + public ReadPageAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task ThrowsFor404() + { + var expected = new StorageException( + new RequestResult { HttpStatusCode = (int)HttpStatusCode.NotFound }, + "Missing.", + inner: null); + LegacyBlob.Setup(x => x.OpenReadAsync(It.IsAny())).Throws(expected); + var url = "https://example/reg/nuget.versioning/0.0.1/0.0.2.json"; + + var actual = await Assert.ThrowsAsync( + () => Target.ReadPageAsync(HiveType.Legacy, url)); + Assert.Same(expected, actual); + } + + [Fact] + public async Task DeserializesGzipped() + { + GzippedBlob.Object.Properties.ContentEncoding = "gzip"; + var commitId = "6c6330d8-6d49-4c51-9a82-398a04f9e448"; + GzippedStream = SerializeToStream(new { commitId }, gzipped: true); + + var index = await Target.ReadPageAsync(HiveType.Gzipped, "https://example/reg-gz/nuget.versioning/0.0.1/0.0.2.json"); + + Assert.Equal(commitId, index.CommitId); + } + + [Fact] + public async Task ProvidesUnencodedStringToStorage() + { + var url = "https://example/reg/测试更新包/0.0.1/0.0.2.json"; + + var index = await Target.ReadPageAsync(HiveType.Legacy, url); + + LegacyContainer.Verify(x => x.GetBlobReference("测试更新包/0.0.1/0.0.2.json"), Times.Once); + } + + [Fact] + public async Task RejectsMismatchingBaseUrl() + { + var url = "https://example/reg-gz/nuget.versioning/0.0.1/0.0.2.json"; + + var ex = await Assert.ThrowsAsync(() => Target.ReadPageAsync(HiveType.Legacy, url)); + Assert.Equal($"URL '{url}' does not start with expected base URL 'https://example/reg/'.", ex.Message); + } + + [Theory] + [MemberData(nameof(AllHivesTestData))] + public async Task DeserializesAllHiveTypes(HiveType hive) + { + var index = await Target.ReadPageAsync(hive, GetBaseUrl(hive) + "nuget.versioning/0.0.1/0.0.2.json"); + + Assert.NotNull(index); + CloudBlobClient.Verify(x => x.GetContainerReference(It.IsAny()), Times.Once); + CloudBlobClient.Verify(x => x.GetContainerReference(GetContainerName(hive)), Times.Once); + var container = GetContainer(hive); + container.Verify(x => x.GetBlobReference(It.IsAny()), Times.Once); + container.Verify(x => x.GetBlobReference("nuget.versioning/0.0.1/0.0.2.json"), Times.Once); + var blob = GetBlob(hive); + blob.Verify(x => x.OpenReadAsync(It.IsAny()), Times.Once); + blob.Verify(x => x.OpenReadAsync(It.Is(a => a.IfMatchETag == null && a.IfNoneMatchETag == null)), Times.Once); + Assert.Equal(blob.Object.Uri.AbsoluteUri, index.Url); + } + } + + public class WriteIndexAsync : Facts + { + public WriteIndexAsync(ITestOutputHelper output) : base(output) + { + LegacyStream = new MemoryStream(); + GzippedStream = new MemoryStream(); + SemVer2Stream = new MemoryStream(); + } + + [Fact] + public async Task SerializesIndex() + { + await Target.WriteIndexAsync(Hive, ReplicaHives, Id, Index); + + var json = Encoding.UTF8.GetString(LegacyStream.ToArray()); + Assert.Equal("{\"commitTimeStamp\":\"0001-01-01T00:00:00+00:00\",\"count\":0}", json); + LegacyContainer.Verify(x => x.GetBlobReference(It.IsAny()), Times.Once); + LegacyContainer.Verify(x => x.GetBlobReference("nuget.versioning/index.json"), Times.Once); + LegacyBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + LegacyBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.Is(a => a.IfMatchETag == null && a.IfNoneMatchETag == null)), Times.Once); + GzippedContainer.Verify(x => x.GetBlobReference(It.IsAny()), Times.Never); + SemVer2Container.Verify(x => x.GetBlobReference(It.IsAny()), Times.Never); + } + + [Fact] + public async Task WritesToReplicaHives() + { + ReplicaHives.Add(HiveType.Gzipped); + ReplicaHives.Add(HiveType.SemVer2); + + await Target.WriteIndexAsync(Hive, ReplicaHives, Id, Index); + LegacyBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + GzippedBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + SemVer2Blob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + EntityBuilder.Verify(x => x.UpdateIndexUrls(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(3)); + EntityBuilder.Verify(x => x.UpdateIndexUrls(Index, HiveType.Legacy, HiveType.Gzipped), Times.Once); + EntityBuilder.Verify(x => x.UpdateIndexUrls(Index, HiveType.Gzipped, HiveType.SemVer2), Times.Once); + EntityBuilder.Verify(x => x.UpdateIndexUrls(Index, HiveType.SemVer2, HiveType.Legacy), Times.Once); + } + + [Theory] + [InlineData(HiveType.Legacy, false)] + [InlineData(HiveType.Gzipped, true)] + [InlineData(HiveType.SemVer2, true)] + public async Task CompressesContent(HiveType hive, bool isGzipped) + { + await Target.WriteIndexAsync(hive, ReplicaHives, Id, Index); + + var stream = GetStream(hive); + var bytes = stream.ToArray(); + if (isGzipped) + { + var output = new MemoryStream(); + using (var content = new MemoryStream(bytes)) + using (var gzipStream = new GZipStream(content, CompressionMode.Decompress)) + { + gzipStream.CopyTo(output); + } + + bytes = output.ToArray(); + } + var json = Encoding.UTF8.GetString(bytes); + Assert.Equal("{\"commitTimeStamp\":\"0001-01-01T00:00:00+00:00\",\"count\":0}", json); + } + + [Theory] + [InlineData(HiveType.Legacy, null)] + [InlineData(HiveType.Gzipped, "gzip")] + [InlineData(HiveType.SemVer2, "gzip")] + public async Task SetsProperties(HiveType hive, string contentEncoding) + { + await Target.WriteIndexAsync(hive, ReplicaHives, Id, Index); + + var blob = GetBlob(hive); + Assert.Equal("application/json", blob.Object.Properties.ContentType); + Assert.Equal("no-store", blob.Object.Properties.CacheControl); + Assert.Equal(contentEncoding, blob.Object.Properties.ContentEncoding); + } + + [Fact] + public async Task SnapshotsBlobWhenConfiguredAndMissingSnapshot() + { + Config.EnsureSingleSnapshot = true; + LegacySegment.Setup(x => x.Results).Returns(() => new[] { LegacyBlob.Object }); + + await Target.WriteIndexAsync(Hive, ReplicaHives, Id, Index); + LegacyBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + LegacyContainer.Verify( + x => x.ListBlobsSegmentedAsync( + "nuget.versioning/index.json", + true, + BlobListingDetails.Snapshots, + 2, + null, + null, + null, + It.IsAny()), + Times.Once); + LegacyBlob.Verify(x => x.SnapshotAsync(It.IsAny()), Times.Once); + } + + [Fact] + public async Task DoesNotSnapshotBlobWhenConfiguredAndAlreadyHasSnapshot() + { + Config.EnsureSingleSnapshot = true; + LegacySegment.Setup(x => x.Results).Returns(() => new[] { LegacyBlob.Object, LegacyBlob.Object }); + + await Target.WriteIndexAsync(Hive, ReplicaHives, Id, Index); + LegacyBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + LegacyContainer.Verify( + x => x.ListBlobsSegmentedAsync( + "nuget.versioning/index.json", + true, + BlobListingDetails.Snapshots, + 2, + null, + null, + null, + It.IsAny()), + Times.Once); + LegacyBlob.Verify(x => x.SnapshotAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task DoesNotListOrSnapshotBlobWhenNotConfiguredToSnapshot() + { + Config.EnsureSingleSnapshot = false; + + await Target.WriteIndexAsync(Hive, ReplicaHives, Id, Index); + LegacyBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + LegacyContainer.Verify( + x => x.ListBlobsSegmentedAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + LegacyBlob.Verify(x => x.SnapshotAsync(It.IsAny()), Times.Never); + } + } + + public class WritePageAsync : Facts + { + public WritePageAsync(ITestOutputHelper output) : base(output) + { + LegacyStream = new MemoryStream(); + GzippedStream = new MemoryStream(); + SemVer2Stream = new MemoryStream(); + Lower = NuGetVersion.Parse("1.0.0"); + Upper = NuGetVersion.Parse("2.0.0"); + } + + public NuGetVersion Lower { get; } + public NuGetVersion Upper { get; } + + [Fact] + public async Task SerializesPage() + { + await Target.WritePageAsync(Hive, ReplicaHives, Id, Lower, Upper, Page); + + var json = Encoding.UTF8.GetString(LegacyStream.ToArray()); + Assert.Equal("{\"commitTimeStamp\":\"0001-01-01T00:00:00+00:00\",\"count\":0}", json); + LegacyContainer.Verify(x => x.GetBlobReference(It.IsAny()), Times.Once); + LegacyContainer.Verify(x => x.GetBlobReference("nuget.versioning/page/1.0.0/2.0.0.json"), Times.Once); + LegacyBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + LegacyBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.Is(a => a.IfMatchETag == null && a.IfNoneMatchETag == null)), Times.Once); + GzippedContainer.Verify(x => x.GetBlobReference(It.IsAny()), Times.Never); + SemVer2Container.Verify(x => x.GetBlobReference(It.IsAny()), Times.Never); + } + + [Fact] + public async Task WritesToReplicaHives() + { + ReplicaHives.Add(HiveType.Gzipped); + ReplicaHives.Add(HiveType.SemVer2); + + await Target.WritePageAsync(Hive, ReplicaHives, Id, Lower, Upper, Page); + LegacyBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + GzippedBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + SemVer2Blob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + EntityBuilder.Verify(x => x.UpdatePageUrls(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(3)); + EntityBuilder.Verify(x => x.UpdatePageUrls(Page, HiveType.Legacy, HiveType.Gzipped), Times.Once); + EntityBuilder.Verify(x => x.UpdatePageUrls(Page, HiveType.Gzipped, HiveType.SemVer2), Times.Once); + EntityBuilder.Verify(x => x.UpdatePageUrls(Page, HiveType.SemVer2, HiveType.Legacy), Times.Once); + } + } + + public class WriteLeafAsync : Facts + { + public WriteLeafAsync(ITestOutputHelper output) : base(output) + { + LegacyStream = new MemoryStream(); + GzippedStream = new MemoryStream(); + SemVer2Stream = new MemoryStream(); + Version = NuGetVersion.Parse("1.0.0"); + } + + public NuGetVersion Version { get; } + + [Fact] + public async Task SerializesPage() + { + await Target.WriteLeafAsync(Hive, ReplicaHives, Id, Version, Leaf); + + var json = Encoding.UTF8.GetString(LegacyStream.ToArray()); + Assert.Equal("{}", json); + LegacyContainer.Verify(x => x.GetBlobReference(It.IsAny()), Times.Once); + LegacyContainer.Verify(x => x.GetBlobReference("nuget.versioning/1.0.0.json"), Times.Once); + LegacyBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + LegacyBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.Is(a => a.IfMatchETag == null && a.IfNoneMatchETag == null)), Times.Once); + GzippedContainer.Verify(x => x.GetBlobReference(It.IsAny()), Times.Never); + SemVer2Container.Verify(x => x.GetBlobReference(It.IsAny()), Times.Never); + } + + [Fact] + public async Task WritesToReplicaHives() + { + ReplicaHives.Add(HiveType.Gzipped); + ReplicaHives.Add(HiveType.SemVer2); + + await Target.WriteLeafAsync(Hive, ReplicaHives, Id, Version, Leaf); + + LegacyBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + GzippedBlob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + SemVer2Blob.Verify(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny()), Times.Once); + EntityBuilder.Verify(x => x.UpdateLeafUrls(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(3)); + EntityBuilder.Verify(x => x.UpdateLeafUrls(Leaf, HiveType.Legacy, HiveType.Gzipped), Times.Once); + EntityBuilder.Verify(x => x.UpdateLeafUrls(Leaf, HiveType.Gzipped, HiveType.SemVer2), Times.Once); + EntityBuilder.Verify(x => x.UpdateLeafUrls(Leaf, HiveType.SemVer2, HiveType.Legacy), Times.Once); + } + } + + public class DeleteIndexAsync : Facts + { + public DeleteIndexAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task DeletesBlobFromHivesAndReplicaHives() + { + ReplicaHives.Add(HiveType.Gzipped); + ReplicaHives.Add(HiveType.SemVer2); + + await Target.DeleteIndexAsync(Hive, ReplicaHives, Id); + + LegacyContainer.Verify(x => x.GetBlobReference("nuget.versioning/index.json"), Times.Once); + GzippedContainer.Verify(x => x.GetBlobReference("nuget.versioning/index.json"), Times.Once); + SemVer2Container.Verify(x => x.GetBlobReference("nuget.versioning/index.json"), Times.Once); + LegacyBlob.Verify(x => x.DeleteIfExistsAsync(), Times.Once); + GzippedBlob.Verify(x => x.DeleteIfExistsAsync(), Times.Once); + SemVer2Blob.Verify(x => x.DeleteIfExistsAsync(), Times.Once); + } + + [Fact] + public async Task DoesNotDeleteNonExistentBlob() + { + LegacyBlob.Setup(x => x.ExistsAsync()).ReturnsAsync(false); + + await Target.DeleteIndexAsync(Hive, ReplicaHives, Id); + + LegacyContainer.Verify(x => x.GetBlobReference("nuget.versioning/index.json"), Times.Once); + LegacyBlob.Verify(x => x.ExistsAsync(), Times.Once); + LegacyBlob.Verify(x => x.DeleteIfExistsAsync(), Times.Never); + } + } + + public class DeleteUrlAsync : Facts + { + public DeleteUrlAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task DeletesBlobFromHivesAndReplicaHives() + { + ReplicaHives.Add(HiveType.Gzipped); + ReplicaHives.Add(HiveType.SemVer2); + + await Target.DeleteUrlAsync(Hive, ReplicaHives, "https://example/reg/nuget.versioning/1.0.0.json"); + + LegacyContainer.Verify(x => x.GetBlobReference("nuget.versioning/1.0.0.json"), Times.Once); + GzippedContainer.Verify(x => x.GetBlobReference("nuget.versioning/1.0.0.json"), Times.Once); + SemVer2Container.Verify(x => x.GetBlobReference("nuget.versioning/1.0.0.json"), Times.Once); + LegacyBlob.Verify(x => x.DeleteIfExistsAsync(), Times.Once); + GzippedBlob.Verify(x => x.DeleteIfExistsAsync(), Times.Once); + SemVer2Blob.Verify(x => x.DeleteIfExistsAsync(), Times.Once); + } + + [Fact] + public async Task DoesNotDeleteNonExistentBlob() + { + LegacyBlob.Setup(x => x.ExistsAsync()).ReturnsAsync(false); + + await Target.DeleteUrlAsync(Hive, ReplicaHives, "https://example/reg/nuget.versioning/1.0.0.json"); + + LegacyContainer.Verify(x => x.GetBlobReference("nuget.versioning/1.0.0.json"), Times.Once); + LegacyBlob.Verify(x => x.ExistsAsync(), Times.Once); + LegacyBlob.Verify(x => x.DeleteIfExistsAsync(), Times.Never); + } + + [Fact] + public async Task RejectsMismatchingBaseUrl() + { + var url = "https://example/reg-gz/nuget.versioning/0.0.1/0.0.2.json"; + + var ex = await Assert.ThrowsAsync( + () => Target.DeleteUrlAsync(Hive, ReplicaHives, url)); + Assert.Equal($"URL '{url}' does not start with expected base URL 'https://example/reg/'.", ex.Message); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + CloudBlobClient = new Mock(); + EntityBuilder = new Mock(); + Throttle = new Mock(); + Options = new Mock>(); + Logger = output.GetLogger(); + + Config = new Catalog2RegistrationConfiguration + { + LegacyBaseUrl = "https://example/reg/", + LegacyStorageContainer = "reg", + GzippedBaseUrl = "https://example/reg-gz/", + GzippedStorageContainer = "reg-gz", + SemVer2BaseUrl = "https://example/reg-gz-semver2/", + SemVer2StorageContainer = "reg-gz-semver2", + EnsureSingleSnapshot = false, + }; + LegacyContainer = new Mock(); + GzippedContainer = new Mock(); + SemVer2Container = new Mock(); + LegacyBlob = new Mock(); + GzippedBlob = new Mock(); + SemVer2Blob = new Mock(); + LegacyStream = new MemoryStream(); + GzippedStream = new MemoryStream(); + SemVer2Stream = new MemoryStream(); + LegacySegment = new Mock(); + Hive = HiveType.Legacy; + ReplicaHives = new List(); + Id = "NuGet.Versioning"; + Index = new RegistrationIndex(); + Page = new RegistrationPage(); + Leaf = new RegistrationLeaf(); + + Options.Setup(x => x.Value).Returns(() => Config); + CloudBlobClient.Setup(x => x.GetContainerReference(Config.LegacyStorageContainer)).Returns(() => LegacyContainer.Object); + CloudBlobClient.Setup(x => x.GetContainerReference(Config.GzippedStorageContainer)).Returns(() => GzippedContainer.Object); + CloudBlobClient.Setup(x => x.GetContainerReference(Config.SemVer2StorageContainer)).Returns(() => SemVer2Container.Object); + LegacyContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(() => LegacyBlob.Object); + GzippedContainer.Setup(x => x.GetBlobReference(It.IsAny())).Returns(() => GzippedBlob.Object); + SemVer2Container.Setup(x => x.GetBlobReference(It.IsAny())).Returns(() => SemVer2Blob.Object); + LegacyBlob.Setup(x => x.Properties).Returns(new BlobProperties()); + LegacyBlob.Setup(x => x.OpenReadAsync(It.IsAny())).ReturnsAsync(() => LegacyStream); + LegacyBlob.Setup(x => x.Uri).Returns(new Uri("https://example/reg/something.json")); + LegacyBlob + .Setup(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask) + .Callback((s, _) => s.CopyTo(LegacyStream)); + LegacyBlob.Setup(x => x.ExistsAsync()).ReturnsAsync(true); + LegacyContainer + .Setup(x => x.ListBlobsSegmentedAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(() => Task.FromResult(LegacySegment.Object)); + LegacySegment.Setup(x => x.Results).Returns(new List()); + GzippedBlob.Setup(x => x.Properties).Returns(new BlobProperties()); + GzippedBlob.Setup(x => x.OpenReadAsync(It.IsAny())).ReturnsAsync(() => GzippedStream); + GzippedBlob.Setup(x => x.Uri).Returns(new Uri("https://example/reg-gz/something.json")); + GzippedBlob + .Setup(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask) + .Callback((s, _) => s.CopyTo(GzippedStream)); + GzippedBlob.Setup(x => x.ExistsAsync()).ReturnsAsync(true); + SemVer2Blob.Setup(x => x.Properties).Returns(new BlobProperties()); + SemVer2Blob.Setup(x => x.OpenReadAsync(It.IsAny())).ReturnsAsync(() => SemVer2Stream); + SemVer2Blob.Setup(x => x.Uri).Returns(new Uri("https://example/reg-gz-semver2/something.json")); + SemVer2Blob + .Setup(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask) + .Callback((s, _) => s.CopyTo(SemVer2Stream)); + SemVer2Blob.Setup(x => x.ExistsAsync()).ReturnsAsync(true); + + SerializeToStream(LegacyStream, new Dictionary { { "@id", LegacyBlob.Object.Uri.AbsoluteUri } }); + SerializeToStream(GzippedStream, new Dictionary { { "@id", GzippedBlob.Object.Uri.AbsoluteUri } }); + SerializeToStream(SemVer2Stream, new Dictionary { { "@id", SemVer2Blob.Object.Uri.AbsoluteUri } }); + + Target = new HiveStorage( + CloudBlobClient.Object, + new RegistrationUrlBuilder(Options.Object), + EntityBuilder.Object, + Throttle.Object, + Options.Object, + Logger); + } + + public Mock CloudBlobClient { get; } + public Mock EntityBuilder { get; } + public Mock Throttle { get; } + public Mock> Options { get; } + public RecordingLogger Logger { get; } + public Catalog2RegistrationConfiguration Config { get; } + public Mock LegacyContainer { get; } + public Mock GzippedContainer { get; } + public Mock SemVer2Container { get; } + public Mock LegacyBlob { get; } + public Mock GzippedBlob { get; } + public Mock SemVer2Blob { get; } + public MemoryStream LegacyStream { get; set; } + public MemoryStream GzippedStream { get; set; } + public MemoryStream SemVer2Stream { get; set; } + public Mock LegacySegment { get; } + public HiveType Hive { get; set; } + public List ReplicaHives { get; } + public string Id { get; } + public RegistrationIndex Index { get; } + public RegistrationPage Page { get; } + public RegistrationLeaf Leaf { get; } + public HiveStorage Target { get; } + + public MemoryStream SerializeToStream(object obj, bool gzipped = false) + { + var memoryStream = new MemoryStream(); + SerializeToStream(memoryStream, obj, gzipped); + return memoryStream; + } + + public void SerializeToStream(MemoryStream stream, object obj, bool gzipped = false) + { + var json = JsonConvert.SerializeObject( + obj, + NuGetJsonSerialization.Settings); + var bytes = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false).GetBytes(json); + + if (gzipped) + { + var memoryStream = new MemoryStream(); + using (memoryStream) + using (var gzip = new GZipStream(memoryStream, CompressionMode.Compress)) + { + gzip.Write(bytes, 0, bytes.Length); + } + + bytes = memoryStream.ToArray(); + } + + stream.Write(bytes, 0, bytes.Length); + stream.Position -= bytes.Length; + } + + public static IEnumerable AllHives => Enum + .GetValues(typeof(HiveType)) + .Cast(); + + public string GetBaseUrl(HiveType hive) => HiveToBaseUrl[hive](this); + public string GetContainerName(HiveType hive) => HiveToContainerName[hive](this); + public Mock GetContainer(HiveType hive) => HiveToContainer[hive](this); + public Mock GetBlob(HiveType hive) => HiveToBlob[hive](this); + public MemoryStream GetStream(HiveType hive) => HiveToStream[hive](this); + + public static IReadOnlyDictionary> HiveToBaseUrl + = new Dictionary> + { + { HiveType.Legacy, c => c.Config.LegacyBaseUrl }, + { HiveType.Gzipped, c => c.Config.GzippedBaseUrl }, + { HiveType.SemVer2, c => c.Config.SemVer2BaseUrl }, + }; + + public static IReadOnlyDictionary> HiveToContainerName + = new Dictionary> + { + { HiveType.Legacy, c => c.Config.LegacyStorageContainer }, + { HiveType.Gzipped, c => c.Config.GzippedStorageContainer }, + { HiveType.SemVer2, c => c.Config.SemVer2StorageContainer }, + }; + + public static IReadOnlyDictionary>> HiveToContainer + = new Dictionary>> + { + { HiveType.Legacy, c => c.LegacyContainer }, + { HiveType.Gzipped, c => c.GzippedContainer }, + { HiveType.SemVer2, c => c.SemVer2Container }, + }; + + public static IReadOnlyDictionary>> HiveToBlob + = new Dictionary>> + { + { HiveType.Legacy, c => c.LegacyBlob }, + { HiveType.Gzipped, c => c.GzippedBlob }, + { HiveType.SemVer2, c => c.SemVer2Blob }, + }; + + public static IReadOnlyDictionary> HiveToStream + = new Dictionary> + { + { HiveType.Legacy, c => c.LegacyStream }, + { HiveType.Gzipped, c => c.GzippedStream }, + { HiveType.SemVer2, c => c.SemVer2Stream }, + }; + + public static IEnumerable AllHivesTestData => AllHives + .Select(x => new object[] { x }); + } + } +} diff --git a/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveUpdaterFacts.cs b/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveUpdaterFacts.cs new file mode 100644 index 000000000..01dadbfdc --- /dev/null +++ b/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/HiveUpdaterFacts.cs @@ -0,0 +1,586 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Packaging.Core; +using NuGet.Protocol.Catalog; +using NuGet.Protocol.Registration; +using NuGet.Services; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class HiveUpdaterFacts + { + public class UpdateAsync : Facts + { + public UpdateAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task RejectsMissingPackageDetailsLeaves() + { + AddPackageDetails("2.0.0"); + EntryToCatalogLeaf.Clear(); + + var ex = await Assert.ThrowsAsync( + () => Target.UpdateAsync(Hive, ReplicaHives, Id, Entries, EntryToCatalogLeaf, RegistrationCommit)); + Assert.Equal("Each PackageDetails catalog commit item must have an associate catalog leaf.", ex.Message); + } + + [Fact] + public async Task RejectsDuplicateVersions() + { + AddPackageDetails("2.0.0"); + AddPackageDetails("2.0.0"); + + var ex = await Assert.ThrowsAsync( + () => Target.UpdateAsync(Hive, ReplicaHives, Id, Entries, EntryToCatalogLeaf, RegistrationCommit)); + Assert.Equal("There must be exactly on catalog commit item per version.", ex.Message); + } + + [Fact] + public async Task RejectsNonSemVer2ReplicaHiveWhenMainHiveIsSemVer2() + { + AddPackageDetails("2.0.0"); + Hive = HiveType.SemVer2; + ReplicaHives.Add(HiveType.Legacy); + + var ex = await Assert.ThrowsAsync( + () => Target.UpdateAsync(Hive, ReplicaHives, Id, Entries, EntryToCatalogLeaf, RegistrationCommit)); + Assert.Equal("A replica hive of a SemVer 2.0.0 hive must also include SemVer 2.0.0.", ex.Message); + } + + [Fact] + public async Task RejectsSemVer2ReplicaHiveWhenMainHiveIsNonSemVer2() + { + AddPackageDetails("2.0.0"); + Hive = HiveType.Legacy; + ReplicaHives.Add(HiveType.SemVer2); + + var ex = await Assert.ThrowsAsync( + () => Target.UpdateAsync(Hive, ReplicaHives, Id, Entries, EntryToCatalogLeaf, RegistrationCommit)); + Assert.Equal("A replica hive of a non-SemVer 2.0.0 hive must also exclude SemVer 2.0.0.", ex.Message); + } + + [Fact] + public async Task MergesCatalogCommitItems() + { + AddPackageDetails("2.0.0"); + + await Target.UpdateAsync(Hive, ReplicaHives, Id, Entries, EntryToCatalogLeaf, RegistrationCommit); + + Merger.Verify( + x => x.MergeAsync(It.IsAny(), It.IsAny>()), + Times.Once); + EntityBuilder.Verify( + x => x.NewLeaf(It.IsAny()), + Times.Once); + EntityBuilder.Verify( + x => x.NewLeaf(It.Is(i => i.CatalogEntry.Version == "2.0.0")), + Times.Once); + EntityBuilder.Verify( + x => x.NewLeaf(RegistrationIndex.Items[0].Items[1]), + Times.Once); + Storage.Verify( + x => x.WriteLeafAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + Storage.Verify( + x => x.WriteLeafAsync(Hive, ReplicaHives, Id, NuGetVersion.Parse("2.0.0"), RegistrationLeaf), + Times.Once); + Storage.Verify( + x => x.WritePageAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + Storage.Verify( + x => x.WriteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny()), + Times.Once); + Storage.Verify( + x => x.WriteIndexAsync(Hive, ReplicaHives, Id, RegistrationIndex), + Times.Once); + Storage.Verify( + x => x.DeleteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny()), + Times.Never); + Storage.Verify( + x => x.DeleteUrlAsync(It.IsAny(), It.IsAny>(), It.IsAny()), + Times.Never); + Assert.Equal("2.0.0", RegistrationIndex.Items[0].Items[1].CatalogEntry.Version); + } + + [Theory] + [InlineData(HiveType.Legacy)] + [InlineData(HiveType.Gzipped)] + public async Task ExcludesSemVer2VersionsFromSemVer1Hives(HiveType hive) + { + Hive = hive; + AddPackageDetails("2.0.0-beta.1"); + + await Target.UpdateAsync(Hive, ReplicaHives, Id, Entries, EntryToCatalogLeaf, RegistrationCommit); + + EntityBuilder.Verify(x => x.NewLeaf(It.IsAny()), Times.Never); + Storage.Verify( + x => x.WriteLeafAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + Storage.Verify( + x => x.WritePageAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + Storage.Verify( + x => x.WriteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny()), + Times.Never); + Assert.Equal(2, RegistrationIndex.Items[0].Items.Count); + Assert.DoesNotContain("2.0.0-beta.1", RegistrationIndex.Items[0].Items.Select(x => x.CatalogEntry.Version)); + } + + [Fact] + public async Task MergesSemVer2VersionsOnSemVer2Hives() + { + AddPackageDetails("2.0.0-beta.1"); + + await Target.UpdateAsync(Hive, ReplicaHives, Id, Entries, EntryToCatalogLeaf, RegistrationCommit); + + EntityBuilder.Verify( + x => x.NewLeaf(It.Is(i => i.CatalogEntry.Version == "2.0.0-beta.1")), + Times.Once); + Storage.Verify( + x => x.WriteLeafAsync(Hive, ReplicaHives, Id, NuGetVersion.Parse("2.0.0-beta.1"), RegistrationLeaf), + Times.Once); + Storage.Verify( + x => x.WriteIndexAsync(Hive, ReplicaHives, Id, RegistrationIndex), + Times.Once); + Assert.Equal("2.0.0-beta.1", RegistrationIndex.Items[0].Items[1].CatalogEntry.Version); + } + + [Fact] + public async Task DeletesEmptyIndex() + { + AddPackageDelete("1.0.0"); + AddPackageDelete("3.0.0"); + + await Target.UpdateAsync(Hive, ReplicaHives, Id, Entries, EntryToCatalogLeaf, RegistrationCommit); + + Storage.Verify( + x => x.WriteLeafAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + Storage.Verify( + x => x.WritePageAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + Storage.Verify( + x => x.WriteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny()), + Times.Never); + Storage.Verify(x => x.DeleteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny()), Times.Once); + Storage.Verify(x => x.DeleteIndexAsync(Hive, ReplicaHives, Id), Times.Once); + Storage.Verify(x => x.DeleteUrlAsync(It.IsAny(), It.IsAny>(), It.IsAny()), Times.Exactly(2)); + Storage.Verify(x => x.DeleteUrlAsync(Hive, ReplicaHives, "https://example/reg/nuget.versioning/1.0.0.json"), Times.Once); + Storage.Verify(x => x.DeleteUrlAsync(Hive, ReplicaHives, "https://example/reg/nuget.versioning/3.0.0.json"), Times.Once); + } + + [Fact] + public async Task WritesExternalizedPage() + { + Config.MaxLeavesPerPage = 2; + Config.MaxInlinedLeafItems = 2; + AddPackageDetails("4.0.0"); + var page = RegistrationIndex.Items[0]; + + await Target.UpdateAsync(Hive, ReplicaHives, Id, Entries, EntryToCatalogLeaf, RegistrationCommit); + + Storage.Verify( + x => x.WriteLeafAsync(Hive, ReplicaHives, Id, NuGetVersion.Parse("4.0.0"), RegistrationLeaf), + Times.Once); + Storage.Verify( + x => x.WritePageAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Exactly(2)); + Storage.Verify( + x => x.WritePageAsync(Hive, ReplicaHives, Id, NuGetVersion.Parse("1.0.0"), NuGetVersion.Parse("3.0.0"), page), + Times.Once); + Storage.Verify( + x => x.WritePageAsync(Hive, ReplicaHives, Id, NuGetVersion.Parse("4.0.0"), NuGetVersion.Parse("4.0.0"), It.IsAny()), + Times.Once); + Storage.Verify( + x => x.WriteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny()), + Times.Once); + Storage.Verify(x => x.WriteIndexAsync(Hive, ReplicaHives, Id, RegistrationIndex), Times.Once); + Storage.Verify(x => x.DeleteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny()), Times.Never); + Storage.Verify(x => x.DeleteUrlAsync(It.IsAny(), It.IsAny>(), It.IsAny()), Times.Never); + Assert.Equal(2, RegistrationIndex.Items.Count); + Assert.Null(RegistrationIndex.Items[0].Items); + Assert.Null(RegistrationIndex.Items[1].Items); + } + + [Fact] + public async Task MovesPageThatIsAlreadyExternal() + { + Config.MaxInlinedLeafItems = 0; + AddPackageDetails("4.0.0"); + var oldPageUrl = "https://example/reg/nuget.versioning/1.0.0/3.0.0.json"; + var newPageUrl = "https://example/reg/nuget.versioning/1.0.0/4.0.0.json"; + var pageItem = RegistrationIndex.Items[0]; + var page = new RegistrationPage + { + Items = pageItem.Items, + }; + pageItem.Url = oldPageUrl; + pageItem.Items = null; + Storage + .Setup(x => x.ReadPageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => page); + + await Target.UpdateAsync(Hive, ReplicaHives, Id, Entries, EntryToCatalogLeaf, RegistrationCommit); + + Storage.Verify(x => x.ReadPageAsync(It.IsAny(), It.IsAny()), Times.Once); + Storage.Verify(x => x.ReadPageAsync(Hive, oldPageUrl), Times.Once); + Storage.Verify( + x => x.WriteLeafAsync(Hive, ReplicaHives, Id, NuGetVersion.Parse("4.0.0"), RegistrationLeaf), + Times.Once); + Storage.Verify( + x => x.WritePageAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + Storage.Verify( + x => x.WritePageAsync(Hive, ReplicaHives, Id, NuGetVersion.Parse("1.0.0"), NuGetVersion.Parse("4.0.0"), page), + Times.Once); + Storage.Verify( + x => x.WriteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny()), + Times.Once); + Storage.Verify(x => x.WriteIndexAsync(Hive, ReplicaHives, Id, RegistrationIndex), Times.Once); + Storage.Verify(x => x.DeleteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny()), Times.Never); + Storage.Verify(x => x.DeleteUrlAsync(It.IsAny(), It.IsAny>(), It.IsAny()), Times.Once); + Storage.Verify(x => x.DeleteUrlAsync(Hive, ReplicaHives, oldPageUrl), Times.Once); + Assert.Equal(newPageUrl, pageItem.Url); + } + + [Fact] + public async Task UpdatesNonInlinedIndex() + { + Config.MaxInlinedLeafItems = 0; + Config.MaxLeavesPerPage = 2; + AddPackageDetails("4.0.0"); + var existingPageItem = RegistrationIndex.Items[0]; + var existingPage = new RegistrationPage + { + Items = existingPageItem.Items, + }; + existingPageItem.Items = null; + + await Target.UpdateAsync(Hive, ReplicaHives, Id, Entries, EntryToCatalogLeaf, RegistrationCommit); + + Storage.Verify(x => x.ReadPageAsync(It.IsAny(), It.IsAny()), Times.Never); + Storage.Verify( + x => x.WriteLeafAsync(Hive, ReplicaHives, Id, NuGetVersion.Parse("4.0.0"), RegistrationLeaf), + Times.Once); + Storage.Verify( + x => x.WritePageAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + Storage.Verify( + x => x.WritePageAsync(Hive, ReplicaHives, Id, NuGetVersion.Parse("4.0.0"), NuGetVersion.Parse("4.0.0"), It.Is(p => p.Items.Single().CatalogEntry.Version == "4.0.0")), + Times.Once); + Storage.Verify( + x => x.WriteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny()), + Times.Once); + Storage.Verify(x => x.WriteIndexAsync(Hive, ReplicaHives, Id, RegistrationIndex), Times.Once); + Storage.Verify(x => x.DeleteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny()), Times.Never); + Storage.Verify(x => x.DeleteUrlAsync(It.IsAny(), It.IsAny>(), It.IsAny()), Times.Never); + Assert.Equal(2, RegistrationIndex.Items.Count); + Assert.Null(RegistrationIndex.Items[0].Items); + Assert.Null(RegistrationIndex.Items[1].Items); + } + + [Fact] + public async Task InlinesPages() + { + AddPackageDetails("4.0.0"); + var oldPageUrl = "https://example/reg/nuget.versioning/1.0.0/3.0.0.json"; + var pageItem = RegistrationIndex.Items[0]; + var page = new RegistrationPage + { + Items = pageItem.Items, + }; + pageItem.Url = oldPageUrl; + pageItem.Items = null; + Storage + .Setup(x => x.ReadPageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => page); + + await Target.UpdateAsync(Hive, ReplicaHives, Id, Entries, EntryToCatalogLeaf, RegistrationCommit); + + Storage.Verify(x => x.ReadPageAsync(It.IsAny(), It.IsAny()), Times.Once); + Storage.Verify(x => x.ReadPageAsync(Hive, oldPageUrl), Times.Once); + Storage.Verify( + x => x.WriteLeafAsync(Hive, ReplicaHives, Id, NuGetVersion.Parse("4.0.0"), RegistrationLeaf), + Times.Once); + Storage.Verify( + x => x.WritePageAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + Storage.Verify( + x => x.WriteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny()), + Times.Once); + Storage.Verify(x => x.WriteIndexAsync(Hive, ReplicaHives, Id, RegistrationIndex), Times.Once); + Storage.Verify(x => x.DeleteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny()), Times.Never); + Storage.Verify(x => x.DeleteUrlAsync(It.IsAny(), It.IsAny>(), It.IsAny()), Times.Once); + Storage.Verify(x => x.DeleteUrlAsync(Hive, ReplicaHives, oldPageUrl), Times.Once); + Assert.NotNull(page.Items); + Assert.Equal(3, page.Items.Count); + Assert.Same(page, Assert.Single(RegistrationIndex.Items)); + } + + /// + /// The count in the index can be out of date if the job crashed before writing the index but after + /// updating a page. This is possible because a page URL can stay the same even if items are added. This + /// can happen if items are added to in the middle of the last page. + /// + [Fact] + public async Task UpdatesNonInlinedPageWithIncorrectCount() + { + Config.MaxInlinedLeafItems = 0; + Config.MaxLeavesPerPage = 4; + + RegistrationIndex.Items[0].Count = 4; + RegistrationIndex.Items[0].Items = null; + + var pageUrl = "https://example/reg/nuget.versioning/4.0.0/7.0.0.json"; + RegistrationIndex.Items.Add(new RegistrationPage + { + Url = pageUrl, + Count = 2, + Lower = "4.0.0", + Upper = "7.0.0", + }); + var page = new RegistrationPage + { + Items = new List + { + new RegistrationLeafItem { CatalogEntry = new RegistrationCatalogEntry { Version = "4.0.0" } }, + new RegistrationLeafItem { CatalogEntry = new RegistrationCatalogEntry { Version = "5.0.0" } }, + new RegistrationLeafItem { CatalogEntry = new RegistrationCatalogEntry { Version = "7.0.0" } }, + }, + }; + Storage + .Setup(x => x.ReadPageAsync(It.IsAny(), pageUrl)) + .ReturnsAsync(() => page); + + AddPackageDetails("6.0.0"); + + await Target.UpdateAsync(Hive, ReplicaHives, Id, Entries, EntryToCatalogLeaf, RegistrationCommit); + + Storage.Verify(x => x.ReadPageAsync(It.IsAny(), It.IsAny()), Times.Once); + Storage.Verify(x => x.ReadPageAsync(Hive, pageUrl), Times.Once); + Storage.Verify( + x => x.WriteLeafAsync(Hive, ReplicaHives, Id, NuGetVersion.Parse("6.0.0"), RegistrationLeaf), + Times.Once); + Storage.Verify( + x => x.WritePageAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + Storage.Verify( + x => x.WritePageAsync(Hive, ReplicaHives, Id, NuGetVersion.Parse("4.0.0"), NuGetVersion.Parse("7.0.0"), page), + Times.Once); + Storage.Verify( + x => x.WriteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny()), + Times.Once); + Storage.Verify(x => x.WriteIndexAsync(Hive, ReplicaHives, Id, RegistrationIndex), Times.Once); + Storage.Verify(x => x.DeleteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny()), Times.Never); + Storage.Verify(x => x.DeleteUrlAsync(It.IsAny(), It.IsAny>(), It.IsAny()), Times.Never); + Assert.Equal(2, RegistrationIndex.Items.Count); + Assert.Null(RegistrationIndex.Items[0].Items); + Assert.Null(RegistrationIndex.Items[1].Items); + } + + [Fact] + public async Task SortsOutOfOrderLeafItems() + { + Config.MaxInlinedLeafItems = 0; + Config.MaxLeavesPerPage = 4; + + var pageUrl = "https://example/reg/nuget.versioning/4.0.0/7.0.0.json"; + RegistrationIndex.Items[0].Url = pageUrl; + RegistrationIndex.Items[0].Count = 2; + RegistrationIndex.Items[0].Items = null; + RegistrationIndex.Items[0].Lower = "4.0.0"; + RegistrationIndex.Items[0].Upper = "7.0.0"; + + var page = new RegistrationPage + { + Items = new List + { + new RegistrationLeafItem { CatalogEntry = new RegistrationCatalogEntry { Version = "5.0.0" } }, + new RegistrationLeafItem { CatalogEntry = new RegistrationCatalogEntry { Version = "4.0.0" } }, + new RegistrationLeafItem { CatalogEntry = new RegistrationCatalogEntry { Version = "7.0.0" } }, + }, + }; + Storage + .Setup(x => x.ReadPageAsync(It.IsAny(), pageUrl)) + .ReturnsAsync(() => page); + + AddPackageDetails("6.0.0"); + + await Target.UpdateAsync(Hive, ReplicaHives, Id, Entries, EntryToCatalogLeaf, RegistrationCommit); + + Storage.Verify(x => x.ReadPageAsync(It.IsAny(), It.IsAny()), Times.Once); + Storage.Verify(x => x.ReadPageAsync(Hive, pageUrl), Times.Once); + Storage.Verify( + x => x.WriteLeafAsync(Hive, ReplicaHives, Id, NuGetVersion.Parse("6.0.0"), RegistrationLeaf), + Times.Once); + Storage.Verify( + x => x.WritePageAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + Storage.Verify( + x => x.WritePageAsync(Hive, ReplicaHives, Id, NuGetVersion.Parse("4.0.0"), NuGetVersion.Parse("7.0.0"), page), + Times.Once); + Storage.Verify( + x => x.WriteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny()), + Times.Once); + Storage.Verify(x => x.WriteIndexAsync(Hive, ReplicaHives, Id, RegistrationIndex), Times.Once); + Storage.Verify(x => x.DeleteIndexAsync(It.IsAny(), It.IsAny>(), It.IsAny()), Times.Never); + Storage.Verify(x => x.DeleteUrlAsync(It.IsAny(), It.IsAny>(), It.IsAny()), Times.Never); + Assert.Null(Assert.Single(RegistrationIndex.Items).Items); + Assert.Equal(new[] { "4.0.0", "5.0.0", "6.0.0", "7.0.0" }, page.Items.Select(x => x.CatalogEntry.Version).ToArray()); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + Storage = new Mock(); + Merger = new Mock(); + EntityBuilder = new Mock(); + Options = new Mock>(); + Logger = output.GetLogger(); + + Config = new Catalog2RegistrationConfiguration(); + Hive = HiveType.SemVer2; + ReplicaHives = new List(); + Id = "NuGet.Versioning"; + Entries = new List(); + EntryToCatalogLeaf = new Dictionary( + ReferenceEqualityComparer.Default); + RegistrationIndex = new RegistrationIndex + { + Items = new List + { + new RegistrationPage + { + Lower = "1.0.0", + Upper = "3.0.0", + Count = 2, + Items = new List + { + new RegistrationLeafItem + { + Url = $"https://example/reg/{Id.ToLowerInvariant()}/1.0.0.json", + CatalogEntry = new RegistrationCatalogEntry + { + Version = "1.0.0", + } + }, + new RegistrationLeafItem + { + Url = $"https://example/reg/{Id.ToLowerInvariant()}/3.0.0.json", + CatalogEntry = new RegistrationCatalogEntry + { + Version = "3.0.0", + } + }, + } + } + } + }; + MergeResult = new HiveMergeResult( + new HashSet(), + new HashSet(), + new HashSet()); + RegistrationLeaf = new RegistrationLeaf(); + RegistrationCommit = new CatalogCommit( + "b580f835-f041-4361-aa46-57e5dc338a63", + new DateTimeOffset(2019, 10, 25, 0, 0, 0, TimeSpan.Zero)); + + Options.Setup(x => x.Value).Returns(() => Config); + Storage + .Setup(x => x.ReadIndexOrNullAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => RegistrationIndex); + var concreteHiveMerger = new HiveMerger(Options.Object, output.GetLogger()); + Merger + .Setup(x => x.MergeAsync(It.IsAny(), It.IsAny>())) + .Returns>((i, e) => concreteHiveMerger.MergeAsync(i, e)); + EntityBuilder + .Setup(x => x.NewLeaf(It.IsAny())) + .Returns(() => RegistrationLeaf); + EntityBuilder + .Setup(x => x.UpdateNonInlinedPageItem( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback((p, h, id, c, l, u) => + { + p.Url = $"https://example/reg/" + + $"{id.ToLowerInvariant()}/" + + $"{l.ToNormalizedString().ToLowerInvariant()}/" + + $"{u.ToNormalizedString().ToLowerInvariant()}.json"; + }); + + Target = new HiveUpdater( + Storage.Object, + Merger.Object, + EntityBuilder.Object, + Options.Object, + Logger); + } + + public Mock Storage { get; } + public Mock Merger { get; } + public Mock EntityBuilder { get; } + public Mock> Options { get; } + public RecordingLogger Logger { get; } + public Catalog2RegistrationConfiguration Config { get; } + public HiveType Hive { get; set; } + public List ReplicaHives { get; } + public string Id { get; } + public List Entries { get; } + public Dictionary EntryToCatalogLeaf { get; } + public RegistrationIndex RegistrationIndex { get; } + public HiveMergeResult MergeResult { get; } + public RegistrationLeaf RegistrationLeaf { get; } + public CatalogCommit RegistrationCommit { get; } + public HiveUpdater Target { get; } + + public void AddPackageDetails(string version) + { + var parsedVersion = NuGetVersion.Parse(version); + var catalogCommitItem = GetCatalogCommitItem(parsedVersion, Schema.DataTypes.PackageDetails); + Entries.Add(catalogCommitItem); + EntryToCatalogLeaf[catalogCommitItem] = new PackageDetailsCatalogLeaf + { + PackageVersion = version, + }; + } + + public void AddPackageDelete(string version) + { + var parsedVersion = NuGetVersion.Parse(version); + var catalogCommitItem = GetCatalogCommitItem(parsedVersion, Schema.DataTypes.PackageDelete); + Entries.Add(catalogCommitItem); + + } + + private CatalogCommitItem GetCatalogCommitItem(NuGetVersion parsedVersion, Uri typeUri) + { + return new CatalogCommitItem( + new Uri($"https://example/catalog/{Entries.Count}/{Id.ToLowerInvariant()}/{parsedVersion.ToNormalizedString().ToLowerInvariant()}.json"), + Entries.Count.ToString(), + new DateTime(2019, 10, 19, 0, 0, 0, DateTimeKind.Utc).AddHours(Entries.Count), + new List(), + new List { typeUri }, + new PackageIdentity(Id, parsedVersion)); + } + } + } +} diff --git a/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/RegistrationUrlBuilderFacts.cs b/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/RegistrationUrlBuilderFacts.cs new file mode 100644 index 000000000..2611b28f2 --- /dev/null +++ b/tests/NuGet.Jobs.Catalog2Registration.Tests/Hives/RegistrationUrlBuilderFacts.cs @@ -0,0 +1,375 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Versioning; +using Xunit; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class RegistrationUrlBuilderFacts + { + public class GetIndexPath : Facts + { + [Fact] + public void EncodesUnsafeCharacters() + { + var path = Target.GetIndexPath("测试更新包"); + + Assert.Equal("%E6%B5%8B%E8%AF%95%E6%9B%B4%E6%96%B0%E5%8C%85/index.json", path); + } + + [Fact] + public void LowercasesId() + { + var path = Target.GetIndexPath("NuGet.Versioning"); + + Assert.Equal("nuget.versioning/index.json", path); + } + } + + public class GetIndexUrl : Facts + { + [Fact] + public void EncodesUnsafeCharacters() + { + var path = Target.GetIndexUrl(HiveType.Legacy, "测试更新包"); + + Assert.Equal("https://example/v3-reg/%E6%B5%8B%E8%AF%95%E6%9B%B4%E6%96%B0%E5%8C%85/index.json", path); + } + + [Fact] + public void LowercasesId() + { + Config.LegacyBaseUrl = "https://example/v3-REG/"; + + var path = Target.GetIndexUrl(HiveType.Legacy, "NuGet.Versioning"); + + Assert.Equal("https://example/v3-REG/nuget.versioning/index.json", path); + } + + [Theory] + [MemberData(nameof(HiveTestData))] + public void HandlesAllBaseUrls(HiveType hive) + { + var baseUrl = GetBaseUrl(hive); + + var path = Target.GetIndexUrl(hive, "NuGet.Versioning"); + + Assert.Equal(baseUrl + "nuget.versioning/index.json", path); + } + } + + public class GetInlinedPageUrl : Facts + { + public GetInlinedPageUrl() + { + Lower = NuGetVersion.Parse("1.0.0"); + Upper = NuGetVersion.Parse("2.0.0"); + } + + public NuGetVersion Lower { get; set; } + public NuGetVersion Upper { get; set; } + + [Fact] + public void EncodesUnsafeCharacters() + { + var path = Target.GetInlinedPageUrl(HiveType.Legacy, "测试更新包", Lower, Upper); + + Assert.Equal("https://example/v3-reg/%E6%B5%8B%E8%AF%95%E6%9B%B4%E6%96%B0%E5%8C%85/index.json#page/1.0.0/2.0.0", path); + } + + [Fact] + public void LowercasesIdAndVersions() + { + Lower = NuGetVersion.Parse("1.0.0-BETA"); + Config.LegacyBaseUrl = "https://example/v3-REG/"; + + var path = Target.GetInlinedPageUrl(HiveType.Legacy, "NuGet.Versioning", Lower, Upper); + + Assert.Equal("https://example/v3-REG/nuget.versioning/index.json#page/1.0.0-beta/2.0.0", path); + } + + [Fact] + public void UsesNormalizedVersion() + { + Lower = NuGetVersion.Parse("1.0.01.0-BETA.1+git"); + + var path = Target.GetInlinedPageUrl(HiveType.Legacy, "NuGet.Versioning", Lower, Upper); + + Assert.Equal("https://example/v3-reg/nuget.versioning/index.json#page/1.0.1-beta.1/2.0.0", path); + } + + [Theory] + [MemberData(nameof(HiveTestData))] + public void HandlesAllBaseUrls(HiveType hive) + { + var baseUrl = GetBaseUrl(hive); + + var path = Target.GetInlinedPageUrl(hive, "NuGet.Versioning", Lower, Upper); + + Assert.Equal(baseUrl + "nuget.versioning/index.json#page/1.0.0/2.0.0", path); + } + } + + public class ConvertHive : Facts + { + [Fact] + public void RejectsMismatchingBaseUrl() + { + var url = "https://example/v3-reg/nuget.versioning/index.json"; + + var ex = Assert.Throws( + () => Target.ConvertHive(HiveType.Gzipped, HiveType.SemVer2, url)); + Assert.Equal($"URL '{url}' does not start with expected base URL 'https://example/v3-reg-gz/'.", ex.Message); + } + + [Fact] + public void ConvertsToSameHive() + { + var url = "https://example/v3-reg/nuget.versioning/index.json"; + + var converted = Target.ConvertHive(HiveType.Legacy, HiveType.Legacy, url); + + Assert.Equal(url, converted); + } + + [Fact] + public void ConvertsToDifferentHive() + { + var url = "https://example/v3-reg/nuget.versioning/index.json"; + var expected = "https://example/v3-reg-gz/nuget.versioning/index.json"; + + var converted = Target.ConvertHive(HiveType.Legacy, HiveType.Gzipped, url); + + Assert.Equal(expected, converted); + } + } + + public class ConvertToPath : Facts + { + [Fact] + public void RejectsMismatchingBaseUrl() + { + var url = "https://example/v3-reg/nuget.versioning/index.json"; + + var ex = Assert.Throws( + () => Target.ConvertToPath(HiveType.Gzipped, url)); + Assert.Equal($"URL '{url}' does not start with expected base URL 'https://example/v3-reg-gz/'.", ex.Message); + } + + [Fact] + public void ConvertsToPath() + { + var expected = "nuget.versioning/index.json"; + var url = "https://example/v3-reg/" + expected; + + var converted = Target.ConvertToPath(HiveType.Legacy, url); + + Assert.Equal(expected, converted); + } + } + + public class GetPageUrl : Facts + { + public GetPageUrl() + { + Lower = NuGetVersion.Parse("1.0.0"); + Upper = NuGetVersion.Parse("2.0.0"); + } + + public NuGetVersion Lower { get; set; } + public NuGetVersion Upper { get; set; } + + [Fact] + public void EncodesUnsafeCharacters() + { + var path = Target.GetPageUrl(HiveType.Legacy, "测试更新包", Lower, Upper); + + Assert.Equal("https://example/v3-reg/%E6%B5%8B%E8%AF%95%E6%9B%B4%E6%96%B0%E5%8C%85/page/1.0.0/2.0.0.json", path); + } + + [Fact] + public void LowercasesIdAndVersions() + { + Lower = NuGetVersion.Parse("1.0.0-BETA"); + Config.LegacyBaseUrl = "https://example/v3-REG/"; + + var path = Target.GetPageUrl(HiveType.Legacy, "NuGet.Versioning", Lower, Upper); + + Assert.Equal("https://example/v3-REG/nuget.versioning/page/1.0.0-beta/2.0.0.json", path); + } + + [Fact] + public void UsesNormalizedVersion() + { + Lower = NuGetVersion.Parse("1.0.01.0-BETA.1+git"); + + var path = Target.GetPageUrl(HiveType.Legacy, "NuGet.Versioning", Lower, Upper); + + Assert.Equal("https://example/v3-reg/nuget.versioning/page/1.0.1-beta.1/2.0.0.json", path); + } + + [Theory] + [MemberData(nameof(HiveTestData))] + public void HandlesAllBaseUrls(HiveType hive) + { + var baseUrl = GetBaseUrl(hive); + + var path = Target.GetPageUrl(hive, "NuGet.Versioning", Lower, Upper); + + Assert.Equal(baseUrl + "nuget.versioning/page/1.0.0/2.0.0.json", path); + } + } + + public class GetPagePath : Facts + { + public GetPagePath() + { + Lower = NuGetVersion.Parse("1.0.0"); + Upper = NuGetVersion.Parse("2.0.0"); + } + + public NuGetVersion Lower { get; set; } + public NuGetVersion Upper { get; set; } + + [Fact] + public void EncodesUnsafeCharacters() + { + var path = Target.GetPagePath("测试更新包", Lower, Upper); + + Assert.Equal("%E6%B5%8B%E8%AF%95%E6%9B%B4%E6%96%B0%E5%8C%85/page/1.0.0/2.0.0.json", path); + } + + [Fact] + public void LowercasesIdAndVersions() + { + Lower = NuGetVersion.Parse("1.0.0-BETA"); + + var path = Target.GetPagePath("NuGet.Versioning", Lower, Upper); + + Assert.Equal("nuget.versioning/page/1.0.0-beta/2.0.0.json", path); + } + + [Fact] + public void UsesNormalizedVersion() + { + Lower = NuGetVersion.Parse("1.0.01.0-BETA.1+git"); + + var path = Target.GetPagePath("NuGet.Versioning", Lower, Upper); + + Assert.Equal("nuget.versioning/page/1.0.1-beta.1/2.0.0.json", path); + } + } + + public class GetLeafPath : Facts + { + [Fact] + public void EncodesUnsafeCharacters() + { + var path = Target.GetLeafPath("测试更新包", NuGetVersion.Parse("1.0.0")); + + Assert.Equal("%E6%B5%8B%E8%AF%95%E6%9B%B4%E6%96%B0%E5%8C%85/1.0.0.json", path); + } + + [Fact] + public void LowercasesIdAndVersions() + { + var path = Target.GetLeafPath("NuGet.Versioning", NuGetVersion.Parse("1.0.0-BETA")); + + Assert.Equal("nuget.versioning/1.0.0-beta.json", path); + } + + [Fact] + public void UsesNormalizedVersion() + { + var path = Target.GetLeafPath("NuGet.Versioning", NuGetVersion.Parse("1.0.01.0-BETA.1+git")); + + Assert.Equal("nuget.versioning/1.0.1-beta.1.json", path); + } + } + + public class GetLeafUrl : Facts + { + [Fact] + public void EncodesUnsafeCharacters() + { + var path = Target.GetLeafUrl(HiveType.Legacy, "测试更新包", NuGetVersion.Parse("1.0.0")); + + Assert.Equal("https://example/v3-reg/%E6%B5%8B%E8%AF%95%E6%9B%B4%E6%96%B0%E5%8C%85/1.0.0.json", path); + } + + [Fact] + public void LowercasesIdAndVersions() + { + Config.LegacyBaseUrl = "https://example/v3-REG/"; + + var path = Target.GetLeafUrl(HiveType.Legacy, "NuGet.Versioning", NuGetVersion.Parse("1.0.0-BETA")); + + Assert.Equal("https://example/v3-REG/nuget.versioning/1.0.0-beta.json", path); + } + + [Fact] + public void UsesNormalizedVersion() + { + var path = Target.GetLeafUrl(HiveType.Legacy, "NuGet.Versioning", NuGetVersion.Parse("1.0.01.0-BETA.1+git")); + + Assert.Equal("https://example/v3-reg/nuget.versioning/1.0.1-beta.1.json", path); + } + + [Theory] + [MemberData(nameof(HiveTestData))] + public void HandlesAllBaseUrls(HiveType hive) + { + var baseUrl = GetBaseUrl(hive); + + var path = Target.GetLeafUrl(hive, "NuGet.Versioning", NuGetVersion.Parse("1.0.0")); + + Assert.Equal(baseUrl + "nuget.versioning/1.0.0.json", path); + } + } + + public abstract class Facts + { + public Facts() + { + Options = new Mock>(); + Config = new Catalog2RegistrationConfiguration + { + LegacyBaseUrl = "https://example/v3-reg/", + GzippedBaseUrl = "https://example/v3-reg-gz/", + SemVer2BaseUrl = "https://example/v3-reg-gz-semver2/", + }; + Options.Setup(x => x.Value).Returns(() => Config); + } + + public Mock> Options { get; } + public Catalog2RegistrationConfiguration Config { get; } + public RegistrationUrlBuilder Target => new RegistrationUrlBuilder(Options.Object); + + public string GetBaseUrl(HiveType hive) + { + switch (hive) + { + case HiveType.Legacy: + return Config.LegacyBaseUrl; + case HiveType.Gzipped: + return Config.GzippedBaseUrl; + case HiveType.SemVer2: + return Config.SemVer2BaseUrl; + default: + throw new NotImplementedException(); + } + } + + public static IEnumerable HiveTestData => Enum + .GetValues(typeof(HiveType)) + .Cast() + .Select(x => new object[] { x }); + } + } +} diff --git a/tests/NuGet.Jobs.Catalog2Registration.Tests/IntegrationTests.cs b/tests/NuGet.Jobs.Catalog2Registration.Tests/IntegrationTests.cs new file mode 100644 index 000000000..b14e99d40 --- /dev/null +++ b/tests/NuGet.Jobs.Catalog2Registration.Tests/IntegrationTests.cs @@ -0,0 +1,451 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Packaging.Core; +using NuGet.Protocol; +using NuGet.Protocol.Catalog; +using NuGet.Services; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class IntegrationTests + { + [Fact] + public async Task AddFirstVersionWhenInlined() + { + // Arrange & Act + await AddVersionsAsync("7.1.2-ALPHA"); + + // Assert + AssertContainerExistence(); + AssertBlobExistence( + existing: new[] + { + "windowsazure.storage/7.1.2-alpha.json", + "windowsazure.storage/index.json", + }, + missing: new string[0]); + } + + [Fact] + public async Task AddSecondVersionWhenInlined() + { + // Arrange + await AddVersionsAsync("7.1.2-ALPHA"); + + // Act + await AddVersionsAsync("7.2.0"); + + // Assert + AssertContainerExistence(); + AssertBlobExistence( + existing: new[] + { + "windowsazure.storage/7.1.2-alpha.json", + "windowsazure.storage/7.2.0.json", + "windowsazure.storage/index.json", + }, + missing: new string[0]); + } + + [Fact] + public async Task UpdateExistingVersion() + { + // Arrange + var version = "7.1.2-alpha"; + await AddVersionsAsync(version); + + // Act + await AddVersionsAsync(version); + + // Assert + AssertContainerExistence(); + AssertBlobExistence( + existing: new[] + { + "windowsazure.storage/7.1.2-alpha.json", + "windowsazure.storage/index.json", + }, + missing: new string[0]); + } + + [Fact] + public async Task DeleteWhenNoVersionExists() + { + // Arrange & Act + await DeleteVersionsAsync("7.1.2-ALPHA"); + + // Assert + AssertContainerExistence(); + AssertBlobExistence( + existing: new string[0], + missing: new[] + { + "windowsazure.storage/index.json", + }); + } + + [Fact] + public async Task DeleteWhenVersionDoesNotExist() + { + // Arrange + await AddVersionsAsync("7.2.0"); + + // Act + await DeleteVersionsAsync("7.1.2-ALPHA"); + + // Assert + AssertContainerExistence(); + AssertBlobExistence( + existing: new[] + { + "windowsazure.storage/7.2.0.json", + "windowsazure.storage/index.json", + }, + missing: new string[0]); + } + + [Fact] + public async Task DeleteLastVersion() + { + // Arrange + await AddVersionsAsync("7.2.0"); + + // Act + await DeleteVersionsAsync("7.2.0"); + + // Assert + AssertContainerExistence(); + AssertBlobExistence( + existing: new string[0], + missing: new[] + { + "windowsazure.storage/7.2.0.json", + "windowsazure.storage/index.json", + }); + } + + [Fact] + public async Task AddItemToTheEndOfAPage() + { + // Arrange + Config.MaxInlinedLeafItems = 0; + await AddVersionsAsync("7.1.2-ALPHA"); + + // Act + await AddVersionsAsync("8.0.0"); + + // Assert + AssertContainerExistence(); + AssertBlobExistence( + existing: new[] + { + "windowsazure.storage/7.1.2-alpha.json", + "windowsazure.storage/8.0.0.json", + "windowsazure.storage/index.json", + "windowsazure.storage/page/7.1.2-alpha/8.0.0.json", + }, + missing: new[] + { + "windowsazure.storage/page/7.1.2-alpha/7.1.2-alpha.json", + }); + } + + [Fact] + public async Task InsertItemInTheMiddleOfAPage() + { + // Arrange + Config.MaxInlinedLeafItems = 0; + await AddVersionsAsync("7.1.2-ALPHA", "8.0.0"); + + // Act + await AddVersionsAsync("7.2.0"); + + // Assert + AssertContainerExistence(); + AssertBlobExistence( + existing: new string[] + { + "windowsazure.storage/7.1.2-alpha.json", + "windowsazure.storage/7.2.0.json", + "windowsazure.storage/8.0.0.json", + "windowsazure.storage/index.json", + "windowsazure.storage/page/7.1.2-alpha/8.0.0.json", + }, + missing: new string[0]); + } + + [Fact] + public async Task DeleteItemInTheMiddleOfAPage() + { + // Arrange + Config.MaxInlinedLeafItems = 0; + await AddVersionsAsync("7.1.2-ALPHA", "7.2.0", "8.0.0"); + + // Act + await DeleteVersionsAsync("7.2.0"); + + // Assert + AssertContainerExistence(); + AssertBlobExistence( + existing: new[] + { + "windowsazure.storage/7.1.2-alpha.json", + "windowsazure.storage/8.0.0.json", + "windowsazure.storage/index.json", + "windowsazure.storage/page/7.1.2-alpha/8.0.0.json", + }, + missing: new[] + { + "windowsazure.storage/7.2.0.json", + }); + } + + [Fact] + public async Task DeleteItemAtTheEndOfAPage() + { + // Arrange + Config.MaxInlinedLeafItems = 0; + await AddVersionsAsync("7.1.2-ALPHA", "8.0.0"); + + // Act + await DeleteVersionsAsync("8.0.0"); + + // Assert + AssertContainerExistence(); + AssertBlobExistence( + existing: new[] + { + "windowsazure.storage/7.1.2-alpha.json", + "windowsazure.storage/index.json", + "windowsazure.storage/page/7.1.2-alpha/7.1.2-alpha.json", + }, + missing: new[] + { + "windowsazure.storage/8.0.0.json", + "windowsazure.storage/page/7.1.2-alpha/8.0.0.json", + }); + } + + [Fact] + public async Task DeleteItemAtTheEndOfANonLastPage() + { + // Arrange + Config.MaxInlinedLeafItems = 0; + Config.MaxLeavesPerPage = 2; + await AddVersionsAsync("7.1.2-ALPHA", "7.2.0", "8.0.0"); + + // Act + await DeleteVersionsAsync("7.2.0"); + + // Assert + AssertContainerExistence(); + AssertBlobExistence( + existing: new[] + { + "windowsazure.storage/7.1.2-alpha.json", + "windowsazure.storage/8.0.0.json", + "windowsazure.storage/index.json", + "windowsazure.storage/page/7.1.2-alpha/8.0.0.json", + }, + missing: new[] + { + "windowsazure.storage/7.2.0.json", + "windowsazure.storage/page/7.1.2-alpha/7.2.0.json", + "windowsazure.storage/page/8.0.0/8.0.0.json", + }); + } + + [Fact] + public async Task DeleteFirstPage() + { + // Arrange + Config.MaxInlinedLeafItems = 0; + Config.MaxLeavesPerPage = 2; + await AddVersionsAsync("7.1.2-ALPHA", "7.2.0", "8.0.0"); + + // Act + await DeleteVersionsAsync("7.1.2-ALPHA", "7.2.0"); + + // Assert + AssertContainerExistence(); + AssertBlobExistence( + existing: new[] + { + "windowsazure.storage/8.0.0.json", + "windowsazure.storage/index.json", + "windowsazure.storage/page/8.0.0/8.0.0.json", + }, + missing: new[] + { + "windowsazure.storage/7.1.2-alpha.json", + "windowsazure.storage/7.2.0.json", + "windowsazure.storage/page/7.1.2-alpha/7.2.0.json", + }); + } + + [Fact] + public async Task DeleteLastVersionWhenNotInlined() + { + // Arrange + Config.MaxInlinedLeafItems = 0; + await AddVersionsAsync("7.1.2-ALPHA"); + + // Act + await DeleteVersionsAsync("7.1.2-ALPHA"); + + // Assert + AssertContainerExistence(); + AssertBlobExistence( + existing: new string[0], + missing: new[] + { + "windowsazure.storage/index.json", + "windowsazure.storage/7.1.2-alpha.json", + "windowsazure.storage/page/7.1.2-alpha/7.1.2-alpha.json", + }); + } + + public IntegrationTests(ITestOutputHelper output) + { + Options = new Mock>(); + Config = new Catalog2RegistrationConfiguration + { + LegacyBaseUrl = "https://example/v3/reg", + LegacyStorageContainer = "v3-reg", + GzippedBaseUrl = "https://example/v3/reg-gz", + GzippedStorageContainer = "v3-reg-gz", + SemVer2BaseUrl = "https://example/v3/reg-gz-semver2", + SemVer2StorageContainer = "v3-reg-gz-semver2", + FlatContainerBaseUrl = "https://example/v3/flatcontainer", + GalleryBaseUrl = "https://example-gallery", + MaxConcurrentHivesPerId = 1, + MaxConcurrentIds = 1, + MaxConcurrentOperationsPerHive = 1, + MaxConcurrentStorageOperations = 1, + EnsureSingleSnapshot = false, + }; + Options.Setup(x => x.Value).Returns(() => Config); + + CloudBlobClient = new InMemoryCloudBlobClient(); + RegistrationUrlBuilder = new RegistrationUrlBuilder(Options.Object); + EntityBuilder = new EntityBuilder(RegistrationUrlBuilder, Options.Object); + Throttle = NullThrottle.Instance; + HiveStorage = new HiveStorage( + CloudBlobClient, + RegistrationUrlBuilder, + EntityBuilder, + Throttle, + Options.Object, + output.GetLogger()); + HiveMerger = new HiveMerger(Options.Object, output.GetLogger()); + HiveUpdater = new HiveUpdater( + HiveStorage, + HiveMerger, + EntityBuilder, + Options.Object, + output.GetLogger()); + RegistrationUpdater = new RegistrationUpdater( + HiveUpdater, + Options.Object, + output.GetLogger()); + } + + public Mock> Options { get; } + public Catalog2RegistrationConfiguration Config { get; } + public InMemoryCloudBlobClient CloudBlobClient { get; } + public RegistrationUrlBuilder RegistrationUrlBuilder { get; } + public EntityBuilder EntityBuilder { get; } + public NullThrottle Throttle { get; } + public HiveStorage HiveStorage { get; } + public HiveMerger HiveMerger { get; } + public HiveUpdater HiveUpdater { get; } + public RegistrationUpdater RegistrationUpdater { get; } + + private void AssertContainerExistence() + { + Assert.Equal( + new[] { Config.LegacyStorageContainer, Config.GzippedStorageContainer, Config.SemVer2StorageContainer }, + CloudBlobClient.Containers.Keys.OrderBy(x => x).ToArray()); + } + + private void AssertBlobExistence(IReadOnlyList existing, IReadOnlyList missing) + { + var allBlobs = existing.Concat(missing).OrderBy(x => x).ToArray(); + Assert.All(CloudBlobClient.Containers.Values, b => Assert.Equal(allBlobs, b.Blobs.Keys.ToArray())); + Assert.All( + CloudBlobClient.Containers.SelectMany(c => c.Value.Blobs).Where(b => !missing.Contains(b.Key)), + b => Assert.True(b.Value.Exists)); + Assert.All( + CloudBlobClient.Containers.SelectMany(c => missing.Select(d => c.Value.Blobs[d])), + b => Assert.False(b.Exists)); + } + + private CatalogCommitItem GetPackageDetailsItem(string id, NuGetVersion version) + { + return new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity(id, version)); + } + + private CatalogCommitItem GetPackageDeleteItem(string id, NuGetVersion version) + { + return new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDelete }, + packageIdentity: new PackageIdentity(id, version)); + } + + private async Task DeleteVersionsAsync(params string[] versions) + { + var entries = new List(); + var entryToLeaf = new Dictionary(); + + foreach (var version in versions) + { + var parsedVersion = NuGetVersion.Parse(version); + var catalogItem = GetPackageDeleteItem(V3Data.PackageId, parsedVersion); + entries.Add(catalogItem); + } + + await RegistrationUpdater.UpdateAsync(V3Data.PackageId, entries, entryToLeaf); + } + + private async Task AddVersionsAsync(params string[] versions) + { + var entries = new List(); + var entryToLeaf = new Dictionary(); + + foreach (var version in versions) + { + var parsedVersion = NuGetVersion.Parse(version); + var catalogItem = GetPackageDetailsItem(V3Data.PackageId, parsedVersion); + var leaf = V3Data.Leaf; + leaf.PackageVersion = parsedVersion.ToFullString(); + leaf.VerbatimVersion = version; + + entries.Add(catalogItem); + entryToLeaf.Add(catalogItem, leaf); + } + + await RegistrationUpdater.UpdateAsync(V3Data.PackageId, entries, entryToLeaf); + } + } +} diff --git a/tests/NuGet.Jobs.Catalog2Registration.Tests/NuGet.Jobs.Catalog2Registration.Tests.csproj b/tests/NuGet.Jobs.Catalog2Registration.Tests/NuGet.Jobs.Catalog2Registration.Tests.csproj new file mode 100644 index 000000000..2bf8d9d6b --- /dev/null +++ b/tests/NuGet.Jobs.Catalog2Registration.Tests/NuGet.Jobs.Catalog2Registration.Tests.csproj @@ -0,0 +1,105 @@ + + + + + Debug + AnyCPU + {296703A3-67BA-4876-8C1D-ACE13DF901EF} + Library + Properties + NuGet.Jobs.Catalog2Registration + NuGet.Jobs.Catalog2Registration.Tests + v4.7.2 + 512 + true + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + 4.10.1 + + + 2.4.1 + + + 2.4.1 + runtime; build; native; contentfiles; analyzers + all + + + + + + + + + + + + + + + + + + + {E97F23B8-ECB0-4AFA-B00C-015C39395FEF} + NuGet.Services.Metadata.Catalog + + + {5abe8807-2209-4948-9fc5-1980a507c47a} + NuGet.Jobs.Catalog2Registration + + + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} + NuGet.Protocol.Catalog + + + {C3F9A738-9759-4B2B-A50D-6507B28A659B} + NuGet.Services.V3 + + + {ccb4d5ef-ac84-449d-ac6e-0a0ad295483a} + NuGet.Services.V3.Tests + + + + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + + + \ No newline at end of file diff --git a/tests/NuGet.Jobs.Catalog2Registration.Tests/Properties/AssemblyInfo.cs b/tests/NuGet.Jobs.Catalog2Registration.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..1d16c9663 --- /dev/null +++ b/tests/NuGet.Jobs.Catalog2Registration.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Jobs.Catalog2Registration.Tests")] +[assembly: ComVisible(false)] +[assembly: Guid("296703a3-67ba-4876-8c1d-ace13df901ef")] diff --git a/tests/NuGet.Jobs.Catalog2Registration.Tests/RegistrationCollectorLogicFacts.cs b/tests/NuGet.Jobs.Catalog2Registration.Tests/RegistrationCollectorLogicFacts.cs new file mode 100644 index 000000000..db79e44dc --- /dev/null +++ b/tests/NuGet.Jobs.Catalog2Registration.Tests/RegistrationCollectorLogicFacts.cs @@ -0,0 +1,241 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Packaging.Core; +using NuGet.Protocol.Catalog; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.V3; +using NuGet.Versioning; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class RegistrationCollectorLogicFacts + { + public class CreateBatchesAsync : Facts + { + public CreateBatchesAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task SingleBatchWithAllItems() + { + var items = new[] + { + new CatalogCommitItem( + uri: null, + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List(), + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + new CatalogCommitItem( + uri: null, + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 2), + types: null, + typeUris: new List(), + packageIdentity: new PackageIdentity("NuGet.Frameworks", NuGetVersion.Parse("2.0.0"))), + }; + + var batches = await Target.CreateBatchesAsync(items); + + var batch = Assert.Single(batches); + Assert.Equal(2, batch.Items.Count); + Assert.Equal(new DateTime(2018, 1, 2), batch.CommitTimeStamp); + Assert.Equal(items[0], batch.Items[0]); + Assert.Equal(items[1], batch.Items[1]); + } + } + + public class OnProcessBatchAsync : Facts + { + public OnProcessBatchAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task DoesNotFetchLeavesForDeleteEntries() + { + var items = new[] + { + new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + new CatalogCommitItem( + uri: new Uri("https://example/1"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 2), + types: null, + typeUris: new List { Schema.DataTypes.PackageDelete }, + packageIdentity: new PackageIdentity("NuGet.Frameworks", NuGetVersion.Parse("2.0.0"))), + }; + + await Target.OnProcessBatchAsync(items); + + CatalogClient.Verify(x => x.GetPackageDetailsLeafAsync("https://example/0"), Times.Once); + CatalogClient.Verify(x => x.GetPackageDetailsLeafAsync(It.IsAny()), Times.Exactly(1)); + CatalogClient.Verify(x => x.GetPackageDeleteLeafAsync(It.IsAny()), Times.Never); + + RegistrationUpdater.Verify( + x => x.UpdateAsync( + "NuGet.Versioning", + It.Is>( + y => y.Count == 1), + It.Is>( + y => y.Count == 1)), + Times.Once); + RegistrationUpdater.Verify( + x => x.UpdateAsync( + "NuGet.Frameworks", + It.Is>( + y => y.Count == 1), + It.Is>( + y => y.Count == 0)), + Times.Once); + } + + [Fact] + public async Task OperatesOnLatestPerPackageIdentityAndGroupsById() + { + var items = new[] + { + new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + new CatalogCommitItem( + uri: new Uri("https://example/1"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 2), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + new CatalogCommitItem( + uri: new Uri("https://example/2"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 2), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("2.0.0"))), + new CatalogCommitItem( + uri: new Uri("https://example/3"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 2), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Frameworks", NuGetVersion.Parse("1.0.0"))), + }; + + await Target.OnProcessBatchAsync(items); + + CatalogClient.Verify(x => x.GetPackageDetailsLeafAsync("https://example/1"), Times.Once); + CatalogClient.Verify(x => x.GetPackageDetailsLeafAsync("https://example/2"), Times.Once); + CatalogClient.Verify(x => x.GetPackageDetailsLeafAsync("https://example/3"), Times.Once); + CatalogClient.Verify(x => x.GetPackageDetailsLeafAsync(It.IsAny()), Times.Exactly(3)); + + RegistrationUpdater.Verify( + x => x.UpdateAsync( + "NuGet.Versioning", + It.Is>( + y => y.Count == 2), + It.Is>( + y => y.Count == 2)), + Times.Once); + RegistrationUpdater.Verify( + x => x.UpdateAsync( + "NuGet.Frameworks", + It.Is>( + y => y.Count == 1), + It.Is>( + y => y.Count == 1)), + Times.Once); + } + + [Fact] + public async Task RejectsMultipleLeavesForTheSamePackageAtTheSameTime() + { + var items = new[] + { + new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + new CatalogCommitItem( + uri: new Uri("https://example/1"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + }; + + var ex = await Assert.ThrowsAsync( + () => Target.OnProcessBatchAsync(items)); + + Assert.Equal( + "There are multiple catalog leaves for a single package at one time.", + ex.Message); + RegistrationUpdater.Verify( + x => x.UpdateAsync( + It.IsAny(), + It.IsAny>(), + It.IsAny>()), + Times.Never); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + CatalogClient = new Mock(); + V3TelemetryService = new Mock(); + CommitCollectorOptions = new Mock>(); + RegistrationUpdater = new Mock(); + Options = new Mock>(); + + CommitCollectorConfiguration = new CommitCollectorConfiguration { MaxConcurrentCatalogLeafDownloads = 1 }; + CommitCollectorOptions.Setup(x => x.Value).Returns(() => CommitCollectorConfiguration); + Configuration = new Catalog2RegistrationConfiguration { MaxConcurrentIds = 1 }; + Options.Setup(x => x.Value).Returns(() => Configuration); + + Target = new RegistrationCollectorLogic( + new CommitCollectorUtility( + CatalogClient.Object, + V3TelemetryService.Object, + CommitCollectorOptions.Object, + output.GetLogger()), + RegistrationUpdater.Object, + Options.Object, + output.GetLogger()); + } + + public Mock CatalogClient { get; } + public Mock V3TelemetryService { get; } + public Mock> CommitCollectorOptions { get; } + public Mock RegistrationUpdater { get; } + public Mock> Options { get; } + public CommitCollectorConfiguration CommitCollectorConfiguration { get; } + public Catalog2RegistrationConfiguration Configuration { get; } + public RegistrationCollectorLogic Target { get; } + } + } +} diff --git a/tests/NuGet.Jobs.Catalog2Registration.Tests/RegistrationUpdaterFacts.cs b/tests/NuGet.Jobs.Catalog2Registration.Tests/RegistrationUpdaterFacts.cs new file mode 100644 index 000000000..077bb4e03 --- /dev/null +++ b/tests/NuGet.Jobs.Catalog2Registration.Tests/RegistrationUpdaterFacts.cs @@ -0,0 +1,116 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Protocol.Catalog; +using NuGet.Services; +using NuGet.Services.Metadata.Catalog; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class RegistrationUpdaterFacts + { + public class UpdateAsync : Facts + { + public UpdateAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task UsesDifferentCommitIdButSameCommitTimestamp() + { + var commits = new ConcurrentBag(); + HiveUpdater + .Setup(x => x.UpdateAsync( + It.IsAny(), + It.IsAny>(), + It.IsAny(), + It.IsAny>(), + It.IsAny>(), + It.IsAny())) + .Returns(Task.CompletedTask) + .Callback, string, IReadOnlyList, IReadOnlyDictionary, CatalogCommit>( + (h, r, i, e, l, c) => commits.Add(c)); + + await Target.UpdateAsync(Id, Entries, EntryToLeaf); + + Assert.Equal(2, commits.Count); + Assert.Single(commits.Select(x => x.Timestamp).Distinct()); + Assert.Equal(2, commits.Select(x => x.Id).Distinct().Count()); + } + + [Fact] + public async Task UsesProperReplicaHives() + { + await Target.UpdateAsync(Id, Entries, EntryToLeaf); + + HiveUpdater.Verify( + x => x.UpdateAsync( + It.IsAny(), + It.IsAny>(), + It.IsAny(), + It.IsAny>(), + It.IsAny>(), + It.IsAny()), + Times.Exactly(2)); + HiveUpdater.Verify( + x => x.UpdateAsync( + HiveType.Legacy, + It.Is>(r => r.Count == 1 && r[0] == HiveType.Gzipped), + Id, + Entries, + EntryToLeaf, + It.IsAny()), + Times.Once); + HiveUpdater.Verify( + x => x.UpdateAsync( + HiveType.SemVer2, + It.Is>(r => r.Count == 0), + Id, + Entries, + EntryToLeaf, + It.IsAny()), + Times.Once); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + HiveUpdater = new Mock(); + Options = new Mock>(); + Logger = output.GetLogger(); + + Config = new Catalog2RegistrationConfiguration(); + Config.MaxConcurrentHivesPerId = 1; + Id = "NuGet.Versioning"; + Entries = new List(); + EntryToLeaf = new Dictionary(); + + Options.Setup(x => x.Value).Returns(() => Config); + + Target = new RegistrationUpdater( + HiveUpdater.Object, + Options.Object, + Logger); + } + + public Mock HiveUpdater { get; } + public Mock> Options { get; } + public RecordingLogger Logger { get; } + public Catalog2RegistrationConfiguration Config { get; } + public string Id { get; } + public List Entries { get; } + public Dictionary EntryToLeaf { get; } + public RegistrationUpdater Target { get; } + } + } +} diff --git a/tests/NuGet.Jobs.Catalog2Registration.Tests/Schema/EntityBuilderFacts.cs b/tests/NuGet.Jobs.Catalog2Registration.Tests/Schema/EntityBuilderFacts.cs new file mode 100644 index 000000000..07ed958c0 --- /dev/null +++ b/tests/NuGet.Jobs.Catalog2Registration.Tests/Schema/EntityBuilderFacts.cs @@ -0,0 +1,970 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Options; +using Moq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NuGet.Protocol.Catalog; +using NuGet.Protocol.Registration; +using NuGet.Services; +using NuGet.Services.V3.Support; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using Xunit; + +namespace NuGet.Jobs.Catalog2Registration +{ + public class EntityBuilderFacts + { + public class UpdateLeafItem : Facts + { + [Fact] + public void EncodesUnsafeCharactersInPackageContentUrl() + { + Target.UpdateLeafItem( + LeafItem, + HiveType.Legacy, + "测试更新包", + PackageDetails); + + Assert.Equal( + "https://example/fc/%E6%B5%8B%E8%AF%95%E6%9B%B4%E6%96%B0%E5%8C%85/7.1.2-alpha/%E6%B5%8B%E8%AF%95%E6%9B%B4%E6%96%B0%E5%8C%85.7.1.2-alpha.nupkg", + LeafItem.PackageContent); + } + + [Fact] + public void UsesEmptyStringForNoTags() + { + var leaf = V3Data.Leaf; + leaf.Tags = null; + + Target.UpdateLeafItem(LeafItem, Hive, Id, leaf); + + Assert.Equal(new[] { string.Empty }, LeafItem.CatalogEntry.Tags.ToArray()); + } + + [Fact] + public void UsesGalleryForLicenseUrlWhenPackageHasLicenseExpression() + { + var leaf = V3Data.Leaf; + leaf.LicenseExpression = "MIT"; + + Target.UpdateLeafItem(LeafItem, Hive, Id, leaf); + + Assert.Equal("MIT", LeafItem.CatalogEntry.LicenseExpression); + Assert.Equal("https://example-gallery/packages/WindowsAzure.Storage/7.1.2-alpha/license", LeafItem.CatalogEntry.LicenseUrl); + } + + [Fact] + public void UsesGalleryForLicenseUrlWhenPackageHasLicenseFile() + { + var leaf = V3Data.Leaf; + leaf.LicenseFile = "license.txt"; + + Target.UpdateLeafItem(LeafItem, Hive, Id, leaf); + + Assert.Equal(string.Empty, LeafItem.CatalogEntry.LicenseExpression); + Assert.Equal("https://example-gallery/packages/WindowsAzure.Storage/7.1.2-alpha/license", LeafItem.CatalogEntry.LicenseUrl); + } + + [Fact] + public void UsesPackageIdCaseFromLeafItemNotParameterWhenBuildingLicenseUrl() + { + var leaf = V3Data.Leaf; + leaf.LicenseExpression = "MIT"; + leaf.PackageId = "WindowsAzure.Storage"; + Id = "windowsazure.storage"; + + Target.UpdateLeafItem(LeafItem, Hive, Id, leaf); + + Assert.Equal("https://example-gallery/packages/WindowsAzure.Storage/7.1.2-alpha/license", LeafItem.CatalogEntry.LicenseUrl); + } + + [Fact] + public void UsesEmptyStringWhenThereIsNoIconUrl() + { + var leaf = V3Data.Leaf; + leaf.IconUrl = null; + + Target.UpdateLeafItem(LeafItem, Hive, Id, leaf); + + Assert.Equal(string.Empty, LeafItem.CatalogEntry.IconUrl); + } + + [Fact] + public void UsesFlatContainerForIconUrlWhenThereIsIconFile() + { + var leaf = V3Data.Leaf; + leaf.IconUrl = null; + leaf.IconFile = "icon.png"; + + Target.UpdateLeafItem(LeafItem, Hive, Id, leaf); + + Assert.Equal("https://example/fc/windowsazure.storage/7.1.2-alpha/icon", LeafItem.CatalogEntry.IconUrl); + } + + [Theory] + [InlineData(HiveType.Legacy, false)] + [InlineData(HiveType.Gzipped, false)] + [InlineData(HiveType.SemVer2, true)] + public void ExcludesDeprecationInformationForNonSemVer2Hives(HiveType hive, bool hasDeprecation) + { + Hive = hive; + + var leaf = V3Data.Leaf; + leaf.Deprecation = new PackageDeprecation + { + Message = "Don't use this for real.", + Reasons = new List { "Other" }, + Url = "https://catalog/#deprecation", + }; + + Target.UpdateLeafItem(LeafItem, Hive, Id, leaf); + + Assert.Equal(hasDeprecation, LeafItem.CatalogEntry.Deprecation != null); + } + + [Fact] + public void PopulatesDeprecationProperties() + { + Hive = HiveType.SemVer2; + var leaf = V3Data.Leaf; + leaf.Deprecation = new PackageDeprecation + { + Message = "Don't use this for real.", + Reasons = new List { "Other", "Legacy" }, + Url = "https://catalog/#deprecation", + AlternatePackage = new AlternatePackage + { + Id = "NuGet.Core", + Range = "[2.8.6, )", + Url = "https://catalog/#alternative" + } + }; + + Target.UpdateLeafItem(LeafItem, Hive, Id, leaf); + + var json = JsonConvert.SerializeObject(LeafItem, SerializerSettings); + Assert.Equal( + @"{ + ""@id"": ""https://example/reg-gz-semver2/windowsazure.storage/7.1.2-alpha.json"", + ""@type"": ""Package"", + ""commitTimeStamp"": ""0001-01-01T00:00:00+00:00"", + ""catalogEntry"": { + ""@type"": ""PackageDetails"", + ""authors"": ""Microsoft"", + ""dependencyGroups"": [ + { + ""dependencies"": [ + { + ""id"": ""Microsoft.Data.OData"", + ""range"": ""[5.6.4, )"", + ""registration"": ""https://example/reg-gz-semver2/microsoft.data.odata/index.json"" + }, + { + ""id"": ""Newtonsoft.Json"", + ""range"": ""[6.0.8, )"", + ""registration"": ""https://example/reg-gz-semver2/newtonsoft.json/index.json"" + } + ], + ""targetFramework"": "".NETFramework4.0-Client"" + } + ], + ""deprecation"": { + ""@id"": ""https://catalog/#deprecation"", + ""@type"": ""deprecation"", + ""alternatePackage"": { + ""@id"": ""https://catalog/#alternative"", + ""@type"": ""alternatePackage"", + ""id"": ""NuGet.Core"", + ""range"": ""[2.8.6, )"" + }, + ""message"": ""Don't use this for real."", + ""reasons"": [ + ""Other"", + ""Legacy"" + ] + }, + ""description"": ""Description."", + ""iconUrl"": ""https://example/fc/windowsazure.storage/7.1.2-alpha/icon"", + ""id"": ""WindowsAzure.Storage"", + ""language"": ""en-US"", + ""licenseExpression"": """", + ""licenseUrl"": ""http://go.microsoft.com/fwlink/?LinkId=331471"", + ""listed"": true, + ""minClientVersion"": ""2.12"", + ""packageContent"": ""https://example/fc/windowsazure.storage/7.1.2-alpha/windowsazure.storage.7.1.2-alpha.nupkg"", + ""projectUrl"": ""https://github.com/Azure/azure-storage-net"", + ""published"": ""2017-01-03T00:00:00+00:00"", + ""requireLicenseAcceptance"": true, + ""summary"": ""Summary."", + ""tags"": [ + ""Microsoft"", + ""Azure"", + ""Storage"", + ""Table"", + ""Blob"", + ""File"", + ""Queue"", + ""Scalable"", + ""windowsazureofficial"" + ], + ""title"": ""Windows Azure Storage"", + ""version"": ""7.1.2-alpha+git"" + }, + ""packageContent"": ""https://example/fc/windowsazure.storage/7.1.2-alpha/windowsazure.storage.7.1.2-alpha.nupkg"", + ""registration"": ""https://example/reg-gz-semver2/windowsazure.storage/index.json"" +}", + json); + } + + [Fact] + public void PopulatesVulnerabilityProperties() + { + Hive = HiveType.SemVer2; + var leaf = V3Data.Leaf; + leaf.Vulnerabilities = new List() { + new PackageVulnerability + { + Id = "https://example/v3/catalog0/data/2020.07.06.06.49.47/bar.1.0.0.json#vulnerability/GitHub/999", + Type = "Vulnerability", + AdvisoryUrl = "https://nvd.nist.gov/vuln/detail/CVE-1234-56789", + Severity = "3" + } + }; + + Target.UpdateLeafItem(LeafItem, Hive, Id, leaf); + + var json = JsonConvert.SerializeObject(LeafItem, SerializerSettings); + Assert.Equal( + @"{ + ""@id"": ""https://example/reg-gz-semver2/windowsazure.storage/7.1.2-alpha.json"", + ""@type"": ""Package"", + ""commitTimeStamp"": ""0001-01-01T00:00:00+00:00"", + ""catalogEntry"": { + ""@type"": ""PackageDetails"", + ""authors"": ""Microsoft"", + ""dependencyGroups"": [ + { + ""dependencies"": [ + { + ""id"": ""Microsoft.Data.OData"", + ""range"": ""[5.6.4, )"", + ""registration"": ""https://example/reg-gz-semver2/microsoft.data.odata/index.json"" + }, + { + ""id"": ""Newtonsoft.Json"", + ""range"": ""[6.0.8, )"", + ""registration"": ""https://example/reg-gz-semver2/newtonsoft.json/index.json"" + } + ], + ""targetFramework"": "".NETFramework4.0-Client"" + } + ], + ""description"": ""Description."", + ""iconUrl"": ""https://example/fc/windowsazure.storage/7.1.2-alpha/icon"", + ""id"": ""WindowsAzure.Storage"", + ""language"": ""en-US"", + ""licenseExpression"": """", + ""licenseUrl"": ""http://go.microsoft.com/fwlink/?LinkId=331471"", + ""listed"": true, + ""minClientVersion"": ""2.12"", + ""packageContent"": ""https://example/fc/windowsazure.storage/7.1.2-alpha/windowsazure.storage.7.1.2-alpha.nupkg"", + ""projectUrl"": ""https://github.com/Azure/azure-storage-net"", + ""published"": ""2017-01-03T00:00:00+00:00"", + ""requireLicenseAcceptance"": true, + ""summary"": ""Summary."", + ""tags"": [ + ""Microsoft"", + ""Azure"", + ""Storage"", + ""Table"", + ""Blob"", + ""File"", + ""Queue"", + ""Scalable"", + ""windowsazureofficial"" + ], + ""title"": ""Windows Azure Storage"", + ""version"": ""7.1.2-alpha+git"", + ""vulnerabilities"": [ + { + ""advisoryUrl"": ""https://nvd.nist.gov/vuln/detail/CVE-1234-56789"", + ""severity"": ""3"" + } + ] + }, + ""packageContent"": ""https://example/fc/windowsazure.storage/7.1.2-alpha/windowsazure.storage.7.1.2-alpha.nupkg"", + ""registration"": ""https://example/reg-gz-semver2/windowsazure.storage/index.json"" +}", + json); + } + } + + public class NewLeaf : Facts + { + [Fact] + public void PopulatesProperties() + { + Target.UpdateLeafItem(LeafItem, Hive, Id, V3Data.Leaf); + + var leaf = Target.NewLeaf(LeafItem); + + var json = JsonConvert.SerializeObject(leaf, SerializerSettings); + Assert.Equal( + @"{ + ""@id"": ""https://example/reg/windowsazure.storage/7.1.2-alpha.json"", + ""@type"": [ + ""Package"", + ""http://schema.nuget.org/catalog#Permalink"" + ], + ""listed"": true, + ""packageContent"": ""https://example/fc/windowsazure.storage/7.1.2-alpha/windowsazure.storage.7.1.2-alpha.nupkg"", + ""published"": ""2017-01-03T00:00:00+00:00"", + ""registration"": ""https://example/reg/windowsazure.storage/index.json"", + ""@context"": { + ""@vocab"": ""http://schema.nuget.org/schema#"", + ""xsd"": ""http://www.w3.org/2001/XMLSchema#"", + ""catalogEntry"": { + ""@type"": ""@id"" + }, + ""registration"": { + ""@type"": ""@id"" + }, + ""packageContent"": { + ""@type"": ""@id"" + }, + ""published"": { + ""@type"": ""xsd:dateTime"" + } + } +}", + json); + } + } + + public class UpdateCommit : Facts + { + [Fact] + public void PopulatesProperties() + { + Target.UpdateCommit(LeafItem, Commit); + + Assert.Equal(V3Data.CommitId, LeafItem.CommitId); + Assert.Equal(V3Data.CommitTimestamp, LeafItem.CommitTimestamp); + } + } + + public class UpdateInlinedPageItem : Facts + { + [Fact] + public void PopulatesProperties() + { + var lower = NuGetVersion.Parse("1.0.0-BETA.1+git"); + var upper = NuGetVersion.Parse("2.0.0+foo"); + Target.UpdateCommit(Page, Commit); + Page.Items = new List { new RegistrationLeafItem() }; + + Target.UpdateInlinedPageItem(Page, Hive, Id, 1, lower, upper); + + var json = JsonConvert.SerializeObject(Page, SerializerSettings); + Assert.Equal(@"{ + ""@id"": ""https://example/reg/windowsazure.storage/index.json#page/1.0.0-beta.1/2.0.0"", + ""@type"": ""catalog:CatalogPage"", + ""commitId"": ""6b9b24dd-7aec-48ae-afc1-2a117e3d50d1"", + ""commitTimeStamp"": ""2018-12-13T12:30:00+00:00"", + ""count"": 1, + ""items"": [ + { + ""commitTimeStamp"": ""0001-01-01T00:00:00+00:00"" + } + ], + ""parent"": ""https://example/reg/windowsazure.storage/index.json"", + ""lower"": ""1.0.0-BETA.1"", + ""upper"": ""2.0.0"" +}", json); + } + } + + public class UpdateNonInlinedPageItem : Facts + { + [Fact] + public void PopulatesProperties() + { + var lower = NuGetVersion.Parse("1.0.0-BETA.1+git"); + var upper = NuGetVersion.Parse("2.0.0+foo"); + Target.UpdateCommit(Page, Commit); + + Target.UpdateNonInlinedPageItem(Page, Hive, Id, 1, lower, upper); + + var json = JsonConvert.SerializeObject(Page, SerializerSettings); + Assert.Equal(@"{ + ""@id"": ""https://example/reg/windowsazure.storage/page/1.0.0-beta.1/2.0.0.json"", + ""@type"": ""catalog:CatalogPage"", + ""commitId"": ""6b9b24dd-7aec-48ae-afc1-2a117e3d50d1"", + ""commitTimeStamp"": ""2018-12-13T12:30:00+00:00"", + ""count"": 1, + ""lower"": ""1.0.0-BETA.1"", + ""upper"": ""2.0.0"" +}", json); + } + } + + public class UpdatePage : Facts + { + [Fact] + public void PopulatesProperties() + { + var lower = NuGetVersion.Parse("1.0.0-BETA.1+git"); + var upper = NuGetVersion.Parse("2.0.0+foo"); + Target.UpdateCommit(Page, Commit); + Page.Items = new List { new RegistrationLeafItem() }; + + Target.UpdatePage(Page, Hive, Id, 1, lower, upper); + + var json = JsonConvert.SerializeObject(Page, SerializerSettings); + Assert.Equal(@"{ + ""@id"": ""https://example/reg/windowsazure.storage/page/1.0.0-beta.1/2.0.0.json"", + ""@type"": ""catalog:CatalogPage"", + ""commitId"": ""6b9b24dd-7aec-48ae-afc1-2a117e3d50d1"", + ""commitTimeStamp"": ""2018-12-13T12:30:00+00:00"", + ""count"": 1, + ""items"": [ + { + ""commitTimeStamp"": ""0001-01-01T00:00:00+00:00"" + } + ], + ""parent"": ""https://example/reg/windowsazure.storage/index.json"", + ""lower"": ""1.0.0-BETA.1"", + ""upper"": ""2.0.0"", + ""@context"": { + ""@vocab"": ""http://schema.nuget.org/schema#"", + ""catalog"": ""http://schema.nuget.org/catalog#"", + ""xsd"": ""http://www.w3.org/2001/XMLSchema#"", + ""items"": { + ""@id"": ""catalog:item"", + ""@container"": ""@set"" + }, + ""commitTimeStamp"": { + ""@id"": ""catalog:commitTimeStamp"", + ""@type"": ""xsd:dateTime"" + }, + ""commitId"": { + ""@id"": ""catalog:commitId"" + }, + ""count"": { + ""@id"": ""catalog:count"" + }, + ""parent"": { + ""@id"": ""catalog:parent"", + ""@type"": ""@id"" + }, + ""tags"": { + ""@id"": ""tag"", + ""@container"": ""@set"" + }, + ""reasons"": { + ""@container"": ""@set"" + }, + ""packageTargetFrameworks"": { + ""@id"": ""packageTargetFramework"", + ""@container"": ""@set"" + }, + ""dependencyGroups"": { + ""@id"": ""dependencyGroup"", + ""@container"": ""@set"" + }, + ""dependencies"": { + ""@id"": ""dependency"", + ""@container"": ""@set"" + }, + ""packageContent"": { + ""@type"": ""@id"" + }, + ""published"": { + ""@type"": ""xsd:dateTime"" + }, + ""registration"": { + ""@type"": ""@id"" + } + } +}", json); + } + } + + public class UpdateIndex : Facts + { + [Fact] + public void PopulatesProperties() + { + Target.UpdateCommit(Index, Commit); + Index.Items = new List() { new RegistrationPage() }; + + Target.UpdateIndex(Index, Hive, Id, 1); + + var json = JsonConvert.SerializeObject(Index, SerializerSettings); + Assert.Equal(@"{ + ""@id"": ""https://example/reg/windowsazure.storage/index.json"", + ""@type"": [ + ""catalog:CatalogRoot"", + ""PackageRegistration"", + ""catalog:Permalink"" + ], + ""commitId"": ""6b9b24dd-7aec-48ae-afc1-2a117e3d50d1"", + ""commitTimeStamp"": ""2018-12-13T12:30:00+00:00"", + ""count"": 1, + ""items"": [ + { + ""commitTimeStamp"": ""0001-01-01T00:00:00+00:00"", + ""count"": 0 + } + ], + ""@context"": { + ""@vocab"": ""http://schema.nuget.org/schema#"", + ""catalog"": ""http://schema.nuget.org/catalog#"", + ""xsd"": ""http://www.w3.org/2001/XMLSchema#"", + ""items"": { + ""@id"": ""catalog:item"", + ""@container"": ""@set"" + }, + ""commitTimeStamp"": { + ""@id"": ""catalog:commitTimeStamp"", + ""@type"": ""xsd:dateTime"" + }, + ""commitId"": { + ""@id"": ""catalog:commitId"" + }, + ""count"": { + ""@id"": ""catalog:count"" + }, + ""parent"": { + ""@id"": ""catalog:parent"", + ""@type"": ""@id"" + }, + ""tags"": { + ""@id"": ""tag"", + ""@container"": ""@set"" + }, + ""reasons"": { + ""@container"": ""@set"" + }, + ""packageTargetFrameworks"": { + ""@id"": ""packageTargetFramework"", + ""@container"": ""@set"" + }, + ""dependencyGroups"": { + ""@id"": ""dependencyGroup"", + ""@container"": ""@set"" + }, + ""dependencies"": { + ""@id"": ""dependency"", + ""@container"": ""@set"" + }, + ""packageContent"": { + ""@type"": ""@id"" + }, + ""published"": { + ""@type"": ""xsd:dateTime"" + }, + ""registration"": { + ""@type"": ""@id"" + } + } +}", json); + } + } + + public class UpdateIndexUrls : Facts + { + [Theory] + [MemberData(nameof(AllHiveTransitionsTestData))] + public void ConvertsHive(HiveType from, HiveType to) + { + // ARRANGE + // These are JSON paths to properties that contain URLs but the URLs don't point to a registration hive + // so they don't need to be converted. + var unconvertedUrls = new[] + { + "@context.@vocab", + "@context.catalog", + "@context.xsd", + "items[0].items[0].catalogEntry.iconUrl", + "items[0].items[0].catalogEntry.licenseUrl", + "items[0].items[0].catalogEntry.packageContent", + "items[0].items[0].catalogEntry.projectUrl", + "items[0].items[0].packageContent", + }.OrderBy(x => x).ToArray(); + + // This is metadata about URLs that point to registration hives and therefore must be converted. + var convertedUrls = new[] + { + new UrlInfo( + "@id", + x => x.Url, + "windowsazure.storage/index.json"), + new UrlInfo( + "items[0].@id", + x => x.Items[0].Url, + "windowsazure.storage/index.json#page/7.1.2-alpha/7.1.2-alpha"), + new UrlInfo( + "items[0].parent", + x => x.Items[0].Parent, + "windowsazure.storage/index.json"), + new UrlInfo( + "items[0].items[0].@id", + x => x.Items[0].Items[0].Url, + "windowsazure.storage/7.1.2-alpha.json"), + new UrlInfo( + "items[0].items[0].registration", + x => x.Items[0].Items[0].Registration, + "windowsazure.storage/index.json"), + new UrlInfo( + "items[0].items[0].catalogEntry.dependencyGroups[0].dependencies[0].registration", + x => x.Items[0].Items[0].CatalogEntry.DependencyGroups[0].Dependencies[0].Registration, + "microsoft.data.odata/index.json"), + new UrlInfo( + "items[0].items[0].catalogEntry.dependencyGroups[0].dependencies[1].registration", + x => x.Items[0].Items[0].CatalogEntry.DependencyGroups[0].Dependencies[1].Registration, + "newtonsoft.json/index.json"), + }; + + var index = InitializeData(from); + + // ACT + Target.UpdateIndexUrls(index, from, to); + + // ASSERT + foreach (var url in convertedUrls) + { + Assert.Equal(GetBaseUrl(to) + url.ExpectedPath, url.GetActualValue(index)); + } + var convertedUrlPaths = convertedUrls.Select(x => x.JsonPath).OrderBy(x => x).ToArray(); + var allUrlPaths = GetJsonPathsForUrlProperties(index); + Assert.Equal(convertedUrlPaths, allUrlPaths.Except(unconvertedUrls).ToArray()); + Assert.Equal(unconvertedUrls, allUrlPaths.Except(convertedUrlPaths).ToArray()); + } + + [Theory] + [MemberData(nameof(AllHiveTransitionsTestData))] + public void DoesNotContainUrlToOldHive(HiveType from, HiveType to) + { + var index = InitializeData(from); + + Target.UpdateIndexUrls(index, from, to); + + var json = JsonConvert.SerializeObject(Index, SerializerSettings); + Assert.DoesNotContain(GetBaseUrl(from), json); + Assert.Contains(GetBaseUrl(to), json); + } + + private RegistrationIndex InitializeData(HiveType hive) + { + Page.Items = new List { LeafItem }; + Index.Items = new List { Page }; + Target.UpdateLeafItem(LeafItem, hive, Id, V3Data.Leaf); + Target.UpdateInlinedPageItem(Page, hive, Id, 1, NuGetVersion.Parse(V3Data.FullVersion), NuGetVersion.Parse(V3Data.FullVersion)); + Target.UpdateIndex(Index, hive, Id, 1); + return Index; + } + } + + public class UpdatePageUrls : Facts + { + [Theory] + [MemberData(nameof(AllHiveTransitionsTestData))] + public void ConvertsHive(HiveType from, HiveType to) + { + // ARRANGE + // These are JSON paths to properties that contain URLs but the URLs don't point to a registration hive + // so they don't need to be converted. + var unconvertedUrls = new[] + { + "@context.@vocab", + "@context.catalog", + "@context.xsd", + "items[0].catalogEntry.iconUrl", + "items[0].catalogEntry.licenseUrl", + "items[0].catalogEntry.packageContent", + "items[0].catalogEntry.projectUrl", + "items[0].packageContent", + }.OrderBy(x => x).ToArray(); + + // This is metadata about URLs that point to registration hives and therefore must be converted. + var convertedUrls = new[] + { + new UrlInfo( + "@id", + x => x.Url, + "windowsazure.storage/page/7.1.2-alpha/7.1.2-alpha.json"), + new UrlInfo( + "parent", + x => x.Parent, + "windowsazure.storage/index.json"), + new UrlInfo( + "items[0].@id", + x => x.Items[0].Url, + "windowsazure.storage/7.1.2-alpha.json"), + new UrlInfo( + "items[0].registration", + x => x.Items[0].Registration, + "windowsazure.storage/index.json"), + new UrlInfo( + "items[0].catalogEntry.dependencyGroups[0].dependencies[0].registration", + x => x.Items[0].CatalogEntry.DependencyGroups[0].Dependencies[0].Registration, + "microsoft.data.odata/index.json"), + new UrlInfo( + "items[0].catalogEntry.dependencyGroups[0].dependencies[1].registration", + x => x.Items[0].CatalogEntry.DependencyGroups[0].Dependencies[1].Registration, + "newtonsoft.json/index.json"), + }; + + var page = InitializeData(from); + + // ACT + Target.UpdatePageUrls(page, from, to); + + // ASSERT + foreach (var url in convertedUrls) + { + Assert.Equal(GetBaseUrl(to) + url.ExpectedPath, url.GetActualValue(page)); + } + var convertedUrlPaths = convertedUrls.Select(x => x.JsonPath).OrderBy(x => x).ToArray(); + var allUrlPaths = GetJsonPathsForUrlProperties(page); + Assert.Equal(convertedUrlPaths, allUrlPaths.Except(unconvertedUrls).ToArray()); + Assert.Equal(unconvertedUrls, allUrlPaths.Except(convertedUrlPaths).ToArray()); + } + + [Theory] + [MemberData(nameof(AllHiveTransitionsTestData))] + public void DoesNotContainUrlToOldHive(HiveType from, HiveType to) + { + var page = InitializeData(from); + + Target.UpdatePageUrls(page, from, to); + + var json = Serialize(page); + Assert.DoesNotContain(GetBaseUrl(from), json); + Assert.Contains(GetBaseUrl(to), json); + } + + private RegistrationPage InitializeData(HiveType hive) + { + Page.Items = new List { LeafItem }; + Target.UpdateLeafItem(LeafItem, hive, Id, V3Data.Leaf); + Target.UpdatePage(Page, hive, Id, 1, NuGetVersion.Parse(V3Data.FullVersion), NuGetVersion.Parse(V3Data.FullVersion)); + return Page; + } + } + + public class UpdateLeafUrls : Facts + { + [Theory] + [MemberData(nameof(AllHiveTransitionsTestData))] + public void ConvertsHive(HiveType from, HiveType to) + { + // ARRANGE + var leaf = InitializeData(from); + + // These are JSON paths to properties that contain URLs but the URLs don't point to a registration hive + // so they don't need to be converted. + var unconvertedUrls = new[] + { + "@context.@vocab", + "@context.xsd", + "@type[1]", + "packageContent", + }.OrderBy(x => x).ToArray(); + + // This is metadata about URLs that point to registration hives and therefore must be converted. + var convertedUrls = new[] + { + new UrlInfo( + "@id", + x => x.Url, + "windowsazure.storage/7.1.2-alpha.json"), + new UrlInfo( + "registration", + x => x.Registration, + "windowsazure.storage/index.json"), + }; + + // ACT + Target.UpdateLeafUrls(leaf, from, to); + + // ASSERT + foreach (var url in convertedUrls) + { + Assert.Equal(GetBaseUrl(to) + url.ExpectedPath, url.GetActualValue(leaf)); + } + var convertedUrlPaths = convertedUrls.Select(x => x.JsonPath).OrderBy(x => x).ToArray(); + var allUrlPaths = GetJsonPathsForUrlProperties(leaf); + Assert.Equal(convertedUrlPaths, allUrlPaths.Except(unconvertedUrls).ToArray()); + Assert.Equal(unconvertedUrls, allUrlPaths.Except(convertedUrlPaths).ToArray()); + } + + [Theory] + [MemberData(nameof(AllHiveTransitionsTestData))] + public void DoesNotContainUrlToOldHive(HiveType from, HiveType to) + { + var leaf = InitializeData(from); + + Target.UpdateLeafUrls(leaf, from, to); + + var json = Serialize(leaf); + Assert.DoesNotContain(GetBaseUrl(from), json); + Assert.Contains(GetBaseUrl(to), json); + } + + private RegistrationLeaf InitializeData(HiveType hive) + { + Target.UpdateLeafItem(LeafItem, hive, Id, V3Data.Leaf); + return Target.NewLeaf(LeafItem); + } + } + + public class UrlInfo + { + public UrlInfo(string jsonPath, Func getActualValue, string expectedPath) + { + JsonPath = jsonPath; + GetActualValue = getActualValue; + ExpectedPath = expectedPath; + } + + public string JsonPath { get; } + public Func GetActualValue { get; } + public string ExpectedPath { get; } + } + + public abstract class Facts + { + public Facts() + { + Options = new Mock>(); + Config = new Catalog2RegistrationConfiguration + { + LegacyBaseUrl = "https://example/reg/", + GzippedBaseUrl = "https://example/reg-gz/", + SemVer2BaseUrl = "https://example/reg-gz-semver2/", + GalleryBaseUrl = "https://example-gallery/", + FlatContainerBaseUrl = "https://example/fc/", + }; + Options.Setup(x => x.Value).Returns(() => Config); + + LeafItem = new RegistrationLeafItem(); + Page = new RegistrationPage(); + Index = new RegistrationIndex(); + Hive = HiveType.Legacy; + Id = V3Data.PackageId; + PackageDetails = new PackageDetailsCatalogLeaf + { + PackageVersion = V3Data.NormalizedVersion, + }; + Commit = new CatalogCommit(V3Data.CommitId, V3Data.CommitTimestamp); + + SerializerSettings = NuGetJsonSerialization.Settings; + SerializerSettings.Formatting = Formatting.Indented; + } + + public string GetBaseUrl(HiveType hive) + { + switch (hive) + { + case HiveType.Legacy: + return Config.LegacyBaseUrl; + case HiveType.Gzipped: + return Config.GzippedBaseUrl; + case HiveType.SemVer2: + return Config.SemVer2BaseUrl; + default: + throw new NotImplementedException(); + } + } + + public Mock> Options { get; } + public Catalog2RegistrationConfiguration Config { get; } + public RegistrationUrlBuilder UrlBuilder => new RegistrationUrlBuilder(Options.Object); + public EntityBuilder Target => new EntityBuilder(UrlBuilder, Options.Object); + public RegistrationLeafItem LeafItem { get; } + public RegistrationPage Page { get; } + public RegistrationIndex Index { get; } + public HiveType Hive { get; set; } + public string Id { get; set; } + public PackageDetailsCatalogLeaf PackageDetails { get; } + public CatalogCommit Commit { get; } + public JsonSerializerSettings SerializerSettings { get; } + + private static readonly IReadOnlyList HiveTypes = Enum + .GetValues(typeof(HiveType)) + .Cast() + .ToList(); + + private static readonly IReadOnlyList> AllHiveTransitions = IterTools + .SubsetsOf(HiveTypes) + .Where(x => x.Count() == 2) + .SelectMany(x => new[] { x.ToList(), x.Reverse().ToList() }) + .Select(x => Tuple.Create(x[0], x[1])) + .ToList(); + + public static IEnumerable AllHiveTransitionsTestData => AllHiveTransitions + .Select(x => new object[] { x.Item1, x.Item2 }); + + /// + /// This is a loose pattern to discover URL strings in a JSON document. It looks for absolute HTTP, HTTPS, + /// schemaless, and some relative URLs. This is not meant to be exhaustive of all possible URL shapes, just + /// ones produced for registration blobs linking to each other. + /// + private static readonly Regex UrlPattern = new Regex("^((https?:)?//|\\.\\.)", RegexOptions.IgnoreCase); + + public string Serialize(T obj) + { + return JsonConvert.SerializeObject(obj, SerializerSettings); + } + + public string[] GetJsonPathsForUrlProperties(T obj) + { + var unparsedJson = Serialize(obj); + var json = JObject.Parse(unparsedJson); + return GetJsonPathsByValuePattern(json, UrlPattern); + } + + private static string[] GetJsonPathsByValuePattern(JObject json, Regex pattern) + { + var output = new List(); + GetJsonPathsByValuePattern(json, pattern, output); + return output.OrderBy(x => x).ToArray(); + } + + private static void GetJsonPathsByValuePattern(JToken json, Regex pattern, List output) + { + if (json == null) + { + return; + } + + if (json.Type == JTokenType.String && pattern.IsMatch((string)json)) + { + output.Add(json.Path); + } + else if (json.Type == JTokenType.Object) + { + foreach (var property in ((JObject)json).Properties()) + { + GetJsonPathsByValuePattern(property.Value, pattern, output); + } + } + else if (json.Type == JTokenType.Array) + { + foreach (var item in (JArray)json) + { + GetJsonPathsByValuePattern(item, pattern, output); + } + } + } + } + } +} diff --git a/tests/NuGet.Protocol.Catalog.Tests/CatalogClientFacts.cs b/tests/NuGet.Protocol.Catalog.Tests/CatalogClientFacts.cs new file mode 100644 index 000000000..21065510e --- /dev/null +++ b/tests/NuGet.Protocol.Catalog.Tests/CatalogClientFacts.cs @@ -0,0 +1,172 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Threading.Tasks; +using Xunit; + +namespace NuGet.Protocol.Catalog +{ + public class CatalogClientFacts + { + public class GetIndexAsync + { + [Fact] + public async Task WorksWithNuGetOrgSnapshot() + { + // Arrange + using (var httpClient = new HttpClient(new TestDataHttpMessageHandler())) + { + var client = GetCatalogClient(httpClient); + + // Act + var actual = await client.GetIndexAsync(TestData.CatalogIndexUrl); + + // Assert + Assert.NotNull(actual); + Assert.NotEqual(default(DateTimeOffset), actual.CommitTimestamp); + Assert.NotEqual(0, actual.Count); + Assert.NotEmpty(actual.Items); + } + } + } + + public class GetPageAsync + { + [Fact] + public async Task WorksWithNuGetOrgSnapshot() + { + // Arrange + using (var httpClient = new HttpClient(new TestDataHttpMessageHandler())) + { + var client = GetCatalogClient(httpClient); + + // Act + var actual = await client.GetPageAsync(TestData.CatalogPageUrl); + + // Assert + Assert.NotNull(actual); + Assert.NotEqual(default(DateTimeOffset), actual.CommitTimestamp); + Assert.NotEqual(0, actual.Count); + Assert.NotEmpty(actual.Items); + } + } + } + + public class GetPackageDeleteLeafAsync + { + [Fact] + public async Task WorksWithNuGetOrgSnapshot() + { + // Arrange + using (var httpClient = new HttpClient(new TestDataHttpMessageHandler())) + { + var client = GetCatalogClient(httpClient); + + // Act + var actual = await client.GetPackageDeleteLeafAsync(TestData.PackageDeleteCatalogLeafUrl); + + // Assert + Assert.NotNull(actual); + } + } + } + + public class GetPackageDetailsLeafAsync + { + [Fact] + public async Task WorksWithNuGetOrgSnapshot() + { + // Arrange + using (var httpClient = new HttpClient(new TestDataHttpMessageHandler())) + { + var client = GetCatalogClient(httpClient); + + // Act + var actual = await client.GetPackageDetailsLeafAsync(TestData.PackageDetailsCatalogLeafUrl); + + // Assert + Assert.NotNull(actual); + } + } + + [Fact] + public async Task ParsesRequiresLicenseAcceptance() + { + // Arrange + using (var httpClient = new HttpClient(new TestDataHttpMessageHandler())) + { + var client = GetCatalogClient(httpClient); + + // Act + var actual = await client.GetPackageDetailsLeafAsync(TestData.PackageDetailsLeafWithRequireLicenseAcceptanceUrl); + + // Assert + Assert.NotNull(actual); + Assert.True(actual.RequireLicenseAcceptance); + } + } + + [Fact] + public async Task WorksWithNuGetOrgDependencyVersionRangeArraySnapshot() + { + // Arrange + using (var httpClient = new HttpClient(new TestDataHttpMessageHandler())) + { + var client = GetCatalogClient(httpClient); + + // Act + var actual = await client.GetPackageDetailsLeafAsync( + TestData.CatalogLeafInvalidDependencyVersionRangeUrl); + + // Assert + Assert.NotNull(actual); + Assert.Equal("[4.0.10, )", actual.DependencyGroups[1].Dependencies[3].Range); + } + } + } + + public class GetLeafAsync + { + [Fact] + public async Task WorksWithNuGetOrgPackageDeleteSnapshot() + { + // Arrange + using (var httpClient = new HttpClient(new TestDataHttpMessageHandler())) + { + var client = GetCatalogClient(httpClient); + + // Act + var actual = await client.GetLeafAsync(TestData.PackageDeleteCatalogLeafUrl); + + // Assert + Assert.NotNull(actual); + } + } + + [Fact] + public async Task WorksWithNuGetOrgPackageDetailsSnapshot() + { + // Arrange + using (var httpClient = new HttpClient(new TestDataHttpMessageHandler())) + { + var client = GetCatalogClient(httpClient); + + // Act + var actual = await client.GetLeafAsync(TestData.PackageDetailsCatalogLeafUrl); + + // Assert + Assert.NotNull(actual); + } + } + } + + private static CatalogClient GetCatalogClient(HttpClient httpClient) + { + return new CatalogClient( + new SimpleHttpClient(httpClient, new NullLogger()), + new NullLogger()); + } + } +} diff --git a/tests/NuGet.Protocol.Catalog.Tests/CatalogProcessorSettingsFacts.cs b/tests/NuGet.Protocol.Catalog.Tests/CatalogProcessorSettingsFacts.cs new file mode 100644 index 000000000..1e2ca7ea8 --- /dev/null +++ b/tests/NuGet.Protocol.Catalog.Tests/CatalogProcessorSettingsFacts.cs @@ -0,0 +1,58 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using FluentAssertions; +using Xunit; + +namespace NuGet.Protocol.Catalog +{ + public class CatalogProcessorSettingsFacts + { + public class Constructor + { + [Fact] + public void HasUnchangingDefaults() + { + // Arrange + var expected = new CatalogProcessorSettings + { + ServiceIndexUrl = "https://api.nuget.org/v3/index.json", + DefaultMinCommitTimestamp = null, + MinCommitTimestamp = DateTimeOffset.MinValue, + MaxCommitTimestamp = DateTimeOffset.MaxValue, + ExcludeRedundantLeaves = true, + }; + + // Act + var actual = new CatalogProcessorSettings(); + + // Assert + actual.Should().BeEquivalentTo(expected); + } + } + + public class Clone + { + [Fact] + public void CopiesAllProperties() + { + // Arrange + var expected = new CatalogProcessorSettings + { + ServiceIndexUrl = "https://example/v3/index.json", + DefaultMinCommitTimestamp = new DateTimeOffset(2017, 11, 8, 13, 50, 44, TimeSpan.Zero), + MinCommitTimestamp = new DateTimeOffset(2010, 1, 1, 0, 0, 0, TimeSpan.Zero), + MaxCommitTimestamp = new DateTimeOffset(2020, 1, 1, 0, 0, 0, TimeSpan.Zero), + ExcludeRedundantLeaves = true, + }; + + // Act + var actual = expected.Clone(); + + // Assert + actual.Should().BeEquivalentTo(expected); + } + } + } +} diff --git a/tests/NuGet.Protocol.Catalog.Tests/Models/CatalogLeafItemFacts.cs b/tests/NuGet.Protocol.Catalog.Tests/Models/CatalogLeafItemFacts.cs new file mode 100644 index 000000000..acc368ce2 --- /dev/null +++ b/tests/NuGet.Protocol.Catalog.Tests/Models/CatalogLeafItemFacts.cs @@ -0,0 +1,30 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; +using Xunit; + +namespace NuGet.Protocol.Catalog +{ + public class CatalogLeafItemFacts + { + [Fact] + public void JsonSerializationException() + { + var leafItem = new CatalogLeafItem + { + Url = "https://api.nuget.org/v3/registration3-gz-semver2/newtonsoft.json/12.0.1.json", + Type = CatalogLeafType.PackageDetails, + CommitId = "47065e84-b83a-434f-9619-1b2f17df91b9", + CommitTimestamp = DateTimeOffset.Parse("2019-10-10T00:00:00.00+00:00"), + PackageId = "Newtonsoft.Json", + PackageVersion = "12.0.1" + }; + + var result = JsonConvert.SerializeObject(leafItem); + + Assert.Equal(TestData.CatalogLeafItem, result); + } + } +} diff --git a/tests/NuGet.Protocol.Catalog.Tests/Models/ModelExtensionsFacts.cs b/tests/NuGet.Protocol.Catalog.Tests/Models/ModelExtensionsFacts.cs new file mode 100644 index 000000000..b8975e2f9 --- /dev/null +++ b/tests/NuGet.Protocol.Catalog.Tests/Models/ModelExtensionsFacts.cs @@ -0,0 +1,225 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Versioning; +using Xunit; + +namespace NuGet.Protocol.Catalog +{ + public class ModelExtensionsFacts + { + public class ParseRange + { + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("\r\n")] + [InlineData(null)] + [InlineData("0.0.0-~4")] + [InlineData("(, )")] + public void ReturnsAllRangeForMissingOrInvalid(string range) + { + var packageDependency = new PackageDependency + { + Range = range, + }; + + var output = packageDependency.ParseRange(); + + Assert.Equal(VersionRange.All, output); + } + } + + public class IsListed + { + [Theory] + [InlineData("1900-01-01Z", false)] + [InlineData("1900-01-01Z", true)] + [InlineData("2018-01-01Z", false)] + [InlineData("2018-01-01Z", true)] + public void PrefersListedPropertyOverPublished(string published, bool listed) + { + var leaf = new PackageDetailsCatalogLeaf + { + Listed = listed, + Published = DateTimeOffset.Parse(published), + }; + + var actual = leaf.IsListed(); + + Assert.Equal(listed, actual); + } + + [Theory] + [InlineData("1899-01-01Z", true)] + [InlineData("1900-01-01Z", false)] + [InlineData("1900-01-01T00:00:01Z", false)] + [InlineData("1900-01-02Z", false)] + [InlineData("1900-02-01Z", false)] + [InlineData("1901-01-01Z", true)] + [InlineData("2018-01-01Z", true)] + public void PrefersListedProperty(string published, bool listed) + { + var leaf = new PackageDetailsCatalogLeaf + { + Published = DateTimeOffset.Parse(published), + }; + + var actual = leaf.IsListed(); + + Assert.Equal(listed, actual); + } + } + + public class IsSemVer2 + { + [Theory] + [InlineData("1.0.0+git", true)] + [InlineData("1.0.0-alpha.1", true)] + [InlineData("1.0.0-alpha.1+git", true)] + [InlineData("1.0.0", false)] + public void SemVer2PackageVersionMeansSemVer2(string packageVersion, bool isSemVer2) + { + var leaf = new PackageDetailsCatalogLeaf + { + PackageVersion = packageVersion, + VerbatimVersion = "1.0.0", + }; + + var actual = leaf.IsSemVer2(); + + Assert.Equal(isSemVer2, actual); + } + + [Fact] + public void AllowsNullVerbatimVersion() + { + var leaf = new PackageDetailsCatalogLeaf + { + PackageVersion = "1.0.0", + DependencyGroups = new List + { + new PackageDependencyGroup + { + Dependencies = new List + { + new PackageDependency + { + Range = "[1.0.0, )", + }, + }, + }, + }, + }; + + var actual = leaf.IsSemVer2(); + + Assert.False(actual); + } + + [Fact] + public void AllowsNullDependencyGroups() + { + var leaf = new PackageDetailsCatalogLeaf + { + PackageVersion = "1.0.0", + VerbatimVersion = "1.0.0", + }; + + var actual = leaf.IsSemVer2(); + + Assert.False(actual); + } + + [Fact] + public void AllowsNullDependencies() + { + var leaf = new PackageDetailsCatalogLeaf + { + PackageVersion = "1.0.0", + VerbatimVersion = "1.0.0", + DependencyGroups = new List + { + new PackageDependencyGroup(), + } + }; + + var actual = leaf.IsSemVer2(); + + Assert.False(actual); + } + + [Theory] + [InlineData("1.0.0+git", true)] + [InlineData("1.0.0-alpha.1", true)] + [InlineData("1.0.0-alpha.1+git", true)] + [InlineData("1.0.0", false)] + public void SemVer2VerbatimVersionMeansSemVer2(string verbatimVersion, bool isSemVer2) + { + var leaf = new PackageDetailsCatalogLeaf + { + PackageVersion = "1.0.0", + VerbatimVersion = verbatimVersion, + }; + + var actual = leaf.IsSemVer2(); + + Assert.Equal(isSemVer2, actual); + } + + + [Theory] + [InlineData("1.0.0+git", true)] + [InlineData("1.0.0-alpha.1", true)] + [InlineData("1.0.0-alpha.1+git", true)] + [InlineData("[1.0.0-alpha.1+git, )", true)] + [InlineData("(, 1.0.0-alpha.1+git)", true)] + [InlineData("(0.0.0+git, 1.0.0-alpha.1+git)", true)] + [InlineData("[1.0.0-alpha.1+git]", true)] + [InlineData("1.0.0", false)] + [InlineData("[1.0.0, )", false)] + [InlineData("(, 1.0.0]", false)] + public void SemVer2DependencyVersionRangeMeansSemVer2(string range, bool isSemVer2) + { + var leaf = new PackageDetailsCatalogLeaf + { + PackageVersion = "1.0.0", + VerbatimVersion = "1.0.0", + DependencyGroups = new List + { + new PackageDependencyGroup + { + Dependencies = new List + { + new PackageDependency + { + Range = "0.0.0", + }, + }, + }, + new PackageDependencyGroup + { + Dependencies = new List + { + new PackageDependency + { + Range = "0.0.1", + }, + new PackageDependency + { + Range = range, + }, + }, + }, + }, + }; + + var actual = leaf.IsSemVer2(); + + Assert.Equal(isSemVer2, actual); + } + } + } +} diff --git a/tests/NuGet.Protocol.Catalog.Tests/NuGet.Protocol.Catalog.Tests.csproj b/tests/NuGet.Protocol.Catalog.Tests/NuGet.Protocol.Catalog.Tests.csproj new file mode 100644 index 000000000..f26369645 --- /dev/null +++ b/tests/NuGet.Protocol.Catalog.Tests/NuGet.Protocol.Catalog.Tests.csproj @@ -0,0 +1,91 @@ + + + + + Debug + AnyCPU + {1F3BC053-796C-4A35-88F4-955A0F142197} + Library + Properties + NuGet.Protocol.Catalog + NuGet.Protocol.Catalog.Tests + v4.7.2 + 512 + true + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + True + True + TestData.resx + + + + + + PublicResXFileCodeGenerator + TestData.Designer.cs + + + + + 5.5.0 + + + 2.2.0 + + + 2.4.1 + + + 2.4.1 + runtime; build; native; contentfiles; analyzers + all + + + + + {d44c2e89-2d98-44bd-8712-8ccbe4e67c9c} + NuGet.Protocol.Catalog + + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + none + + + \ No newline at end of file diff --git a/tests/NuGet.Protocol.Catalog.Tests/NullLogger.cs b/tests/NuGet.Protocol.Catalog.Tests/NullLogger.cs new file mode 100644 index 000000000..1a0b55571 --- /dev/null +++ b/tests/NuGet.Protocol.Catalog.Tests/NullLogger.cs @@ -0,0 +1,40 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Extensions.Logging; + +namespace NuGet.Protocol.Catalog +{ + /// + /// Source: https://github.com/aspnet/Extensions/blob/815526c45fed4cbdcf037522f504149a01147975/src/Logging/Logging.Abstractions/src/NullLoggerT.cs + /// + internal class NullLogger : ILogger + { + public IDisposable BeginScope(TState state) + { + return new NullDisposable(); + } + + public bool IsEnabled(LogLevel logLevel) + { + return false; + } + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception exception, + Func formatter) + { + } + + private class NullDisposable : IDisposable + { + public void Dispose() + { + } + } + } +} \ No newline at end of file diff --git a/tests/NuGet.Protocol.Catalog.Tests/Properties/AssemblyInfo.cs b/tests/NuGet.Protocol.Catalog.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..0110b11e2 --- /dev/null +++ b/tests/NuGet.Protocol.Catalog.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Protocol.Catalog.Tests")] +[assembly: ComVisible(false)] +[assembly: Guid("1f3bc053-796c-4a35-88f4-955a0f142197")] diff --git a/tests/NuGet.Protocol.Catalog.Tests/TestData.Designer.cs b/tests/NuGet.Protocol.Catalog.Tests/TestData.Designer.cs new file mode 100644 index 000000000..1bab2aaec --- /dev/null +++ b/tests/NuGet.Protocol.Catalog.Tests/TestData.Designer.cs @@ -0,0 +1,316 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace NuGet.Protocol.Catalog { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + public class TestData { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal TestData() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + public static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("NuGet.Protocol.Catalog.TestData", typeof(TestData).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + public static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/v3/catalog0/index.json","@type":["CatalogRoot","AppendOnlyCatalog","Permalink"],"commitId":"57de6c98-d4c6-4a24-95b9-1829c5013985","commitTimeStamp":"2017-11-06T19:30:56.0421411Z","count":2945,"nuget:lastCreated":"2017-11-06T19:30:30.19Z","nuget:lastDeleted":"2017-11-06T19:27:45.3684766Z","nuget:lastEdited":"2017-11-06T19:02:37.87Z","items":[{"@id":"https://api.nuget.org/v3/catalog0/page0.json","@type":"CatalogPage","commitId":"00000000-0000-0000-0000-000000000000","commitTimeSt [rest of string was truncated]";. + /// + public static string CatalogIndex { + get { + return ResourceManager.GetString("CatalogIndex", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to https://api.nuget.org/v3/catalog0/index.json. + /// + public static string CatalogIndexUrl { + get { + return ResourceManager.GetString("CatalogIndexUrl", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json", + /// "@type": [ + /// "PackageDetails", + /// "catalog:Permalink" + /// ], + /// "authors": "3344hp", + /// "catalog:commitId": "eddb29f8-32c6-41da-8928-0940927a708b", + /// "catalog:commitTimeStamp": "2016-02-21T11:06:01.8896907Z", + /// "created": "2016-02-21T11:05:37.54Z", + /// "description": "Dingu.Generic.Repo.EF7 Class Library", + /// "frameworkAssemblyGroup": { + /// "@id": "https://api.nuget.org/v3/catalog0/data [rest of string was truncated]";. + /// + public static string CatalogLeafInvalidDependencyVersionRange { + get { + return ResourceManager.GetString("CatalogLeafInvalidDependencyVersionRange", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json. + /// + public static string CatalogLeafInvalidDependencyVersionRangeUrl { + get { + return ResourceManager.GetString("CatalogLeafInvalidDependencyVersionRangeUrl", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/v3/registration3-gz-semver2/newtonsoft.json/12.0.1.json","@type":"nuget:PackageDetails","commitId":"47065e84-b83a-434f-9619-1b2f17df91b9","commitTimeStamp":"2019-10-10T00:00:00+00:00","nuget:id":"Newtonsoft.Json","nuget:version":"12.0.1"}. + /// + public static string CatalogLeafItem { + get { + return ResourceManager.GetString("CatalogLeafItem", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/v3/catalog0/page2944.json","@type":"CatalogPage","commitId":"f241ce46-35ba-44c2-bd72-790eb44539a5","commitTimeStamp":"2017-11-06T22:07:49.3270578Z","count":218,"parent":"https://api.nuget.org/v3/catalog0/index.json","items":[{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.container.1.3.2.json","@type":"nuget:PackageDetails","commitId":"f241ce46-35ba-44c2-bd72-790eb44539a5","commitTimeStamp":"2017-11-06T22:07:49.3270578Z","nuget:id":"Dotnettency.Co [rest of string was truncated]";. + /// + public static string CatalogPage { + get { + return ResourceManager.GetString("CatalogPage", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to https://api.nuget.org/v3/catalog0/page2944.json. + /// + public static string CatalogPageUrl { + get { + return ResourceManager.GetString("CatalogPageUrl", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "https://api.nuget.org/v3/catalog0/data/2017.11.06.19.29.03/microsoft.azure.iot.edge.function.0.3.0.json", + /// "@type": [ + /// "PackageDelete", + /// "catalog:Permalink" + /// ], + /// "catalog:commitId": "b429c4e3-5127-4430-9cc8-74927c9c3886", + /// "catalog:commitTimeStamp": "2017-11-06T19:29:03.6198426Z", + /// "id": "Microsoft.Azure.IoT.Edge.Function", + /// "originalId": "Microsoft.Azure.IoT.Edge.Function", + /// "published": "2017-11-06T19:27:45.3684766Z", + /// "version": "0.3.0", + /// "@context": { + /// "@vocab" [rest of string was truncated]";. + /// + public static string PackageDeleteCatalogLeaf { + get { + return ResourceManager.GetString("PackageDeleteCatalogLeaf", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to https://api.nuget.org/v3/catalog0/data/2017.11.06.19.29.03/microsoft.azure.iot.edge.function.0.3.0.json. + /// + public static string PackageDeleteCatalogLeafUrl { + get { + return ResourceManager.GetString("PackageDeleteCatalogLeafUrl", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.container.1.3.2.json", + /// "@type": [ + /// "PackageDetails", + /// "catalog:Permalink" + /// ], + /// "authors": "Darrell Tunnell", + /// "catalog:commitId": "f241ce46-35ba-44c2-bd72-790eb44539a5", + /// "catalog:commitTimeStamp": "2017-11-06T22:07:49.3270578Z", + /// "created": "2017-11-06T22:06:53.64Z", + /// "description": "Container support, for the dotnettency Mutlitenancy library for dotnet standard compatible applications.", + /// "id": "Do [rest of string was truncated]";. + /// + public static string PackageDetailsCatalogLeaf { + get { + return ResourceManager.GetString("PackageDetailsCatalogLeaf", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.container.1.3.2.json. + /// + public static string PackageDetailsCatalogLeafUrl { + get { + return ResourceManager.GetString("PackageDetailsCatalogLeafUrl", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to { + /// "@id": "https://api.nuget.org/v3/catalog0/data/2015.02.01.06.22.45/antixss.4.0.1.json", + /// "@type": [ + /// "PackageDetails", + /// "catalog:Permalink" + /// ], + /// "authors": "Microsoft", + /// "catalog:commitId": "b3f4fc8a-7522-42a3-8fee-a91d5488c0b1", + /// "catalog:commitTimeStamp": "2015-02-01T06:22:45.8488496Z", + /// "created": "2011-01-07T07:49:50.307Z", + /// "description": "AntiXSS is an encoding library which uses a safe list approach to encoding. It provides Html, XML, Url, Form, LDAP, CSS, JScript and VBScr [rest of string was truncated]";. + /// + public static string PackageDetailsLeafWithRequireLicenseAcceptance { + get { + return ResourceManager.GetString("PackageDetailsLeafWithRequireLicenseAcceptance", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to https://api.nuget.org/v3/catalog0/data/2015.02.01.06.22.45/antixss.4.0.1.json. + /// + public static string PackageDetailsLeafWithRequireLicenseAcceptanceUrl { + get { + return ResourceManager.GetString("PackageDetailsLeafWithRequireLicenseAcceptanceUrl", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/index.json","@type":["catalog:CatalogRoot","PackageRegistration","catalog:Permalink"],"commitId":"bef2767e-f5ae-4713-b5fc-510945bacdf9","commitTimeStamp":"2018-11-26T19:53:07.9393299Z","count":1,"items":[{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/index.json#page/0.1.1/0.3.1","@type":"catalog:CatalogPage","commitId":"bef2767e-f5ae-4713-b5fc-510945bacdf9","commitTimeStamp":"2018-11-26T19:53:07.9393299Z","c [rest of string was truncated]";. + /// + public static string RegistrationIndexInlinedItems { + get { + return ResourceManager.GetString("RegistrationIndexInlinedItems", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/index.json. + /// + public static string RegistrationIndexInlinedItemsUrl { + get { + return ResourceManager.GetString("RegistrationIndexInlinedItemsUrl", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json","@type":["catalog:CatalogRoot","PackageRegistration","catalog:Permalink"],"commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":19,"items":[{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/1.0.728-unstable/1.0.965-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08 [rest of string was truncated]";. + /// + public static string RegistrationIndexWithoutInlinedItems { + get { + return ResourceManager.GetString("RegistrationIndexWithoutInlinedItems", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json. + /// + public static string RegistrationIndexWithoutInlinedItemsUrl { + get { + return ResourceManager.GetString("RegistrationIndexWithoutInlinedItemsUrl", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/v3/registration3-gz-semver2/newtonsoft.json/12.0.1.json","@type":["Package","http://schema.nuget.org/catalog#Permalink"],"catalogEntry":"https://api.nuget.org/v3/catalog0/data/2018.11.27.18.15.55/newtonsoft.json.12.0.1.json","listed":true,"packageContent":"https://api.nuget.org/v3-flatcontainer/newtonsoft.json/12.0.1/newtonsoft.json.12.0.1.nupkg","published":"2018-11-27T18:11:37.08+00:00","registration":"https://api.nuget.org/v3/registration3-gz-semver2/newtonsoft.json/index.js [rest of string was truncated]";. + /// + public static string RegistrationLeafListed { + get { + return ResourceManager.GetString("RegistrationLeafListed", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to https://api.nuget.org/v3/registration3-gz-semver2/newtonsoft.json/12.0.1.json. + /// + public static string RegistrationLeafListedUrl { + get { + return ResourceManager.GetString("RegistrationLeafListedUrl", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/0.1.1.json","@type":["Package","http://schema.nuget.org/catalog#Permalink"],"catalogEntry":"https://api.nuget.org/v3/catalog0/data/2018.11.13.04.43.04/microbuild.core.0.1.1.json","listed":false,"packageContent":"https://api.nuget.org/v3-flatcontainer/microbuild.core/0.1.1/microbuild.core.0.1.1.nupkg","published":"1900-01-01T00:00:00+00:00","registration":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/index.json","@ [rest of string was truncated]";. + /// + public static string RegistrationLeafUnlisted { + get { + return ResourceManager.GetString("RegistrationLeafUnlisted", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/0.1.1.json. + /// + public static string RegistrationLeafUnlistedUrl { + get { + return ResourceManager.GetString("RegistrationLeafUnlistedUrl", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to {"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/3.5.35078-unstable/3.5.35142-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":55,"lower":"3.5.35078-Unstable","parent":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json","upper":"3.5.35142-Unstable","items":[{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35078-unstabl [rest of string was truncated]";. + /// + public static string RegistrationPage { + get { + return ResourceManager.GetString("RegistrationPage", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/3.5.35078-unstable/3.5.35142-unstable.json. + /// + public static string RegistrationPageUrl { + get { + return ResourceManager.GetString("RegistrationPageUrl", resourceCulture); + } + } + } +} diff --git a/tests/NuGet.Protocol.Catalog.Tests/TestData.resx b/tests/NuGet.Protocol.Catalog.Tests/TestData.resx new file mode 100644 index 000000000..a80445d39 --- /dev/null +++ b/tests/NuGet.Protocol.Catalog.Tests/TestData.resx @@ -0,0 +1,573 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + {"@id":"https://api.nuget.org/v3/catalog0/index.json","@type":["CatalogRoot","AppendOnlyCatalog","Permalink"],"commitId":"57de6c98-d4c6-4a24-95b9-1829c5013985","commitTimeStamp":"2017-11-06T19:30:56.0421411Z","count":2945,"nuget:lastCreated":"2017-11-06T19:30:30.19Z","nuget:lastDeleted":"2017-11-06T19:27:45.3684766Z","nuget:lastEdited":"2017-11-06T19:02:37.87Z","items":[{"@id":"https://api.nuget.org/v3/catalog0/page0.json","@type":"CatalogPage","commitId":"00000000-0000-0000-0000-000000000000","commitTimeStamp":"2015-02-01T06:30:11.7477681Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1.json","@type":"CatalogPage","commitId":"8bcd3cbf-74f0-47a2-a7ae-b7ecc50005d3","commitTimeStamp":"2015-02-01T06:39:53.9553899Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2.json","@type":"CatalogPage","commitId":"ddbb307f-3af3-4276-aac8-04577b703a09","commitTimeStamp":"2015-02-01T06:49:12.657797Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page3.json","@type":"CatalogPage","commitId":"3a94eead-7619-46b6-a502-13d7a21d9a97","commitTimeStamp":"2015-02-01T06:58:47.9532886Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page4.json","@type":"CatalogPage","commitId":"70e23de0-2243-457a-99fc-d099eb291a01","commitTimeStamp":"2015-02-01T07:06:41.27878Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page5.json","@type":"CatalogPage","commitId":"22e1f6ed-d412-421e-b476-1b24ee2b7b23","commitTimeStamp":"2015-02-01T07:15:44.5527543Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page6.json","@type":"CatalogPage","commitId":"423a2cbb-f351-4e47-bb51-08fe58c4ca55","commitTimeStamp":"2015-02-01T07:24:24.5537913Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page7.json","@type":"CatalogPage","commitId":"3f38b04c-d916-42c8-9128-dd2421c0a757","commitTimeStamp":"2015-02-01T07:33:28.9218723Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page8.json","@type":"CatalogPage","commitId":"05aa7e6c-cb25-4e09-8456-cc263f0c4906","commitTimeStamp":"2015-02-01T07:44:56.5453045Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page9.json","@type":"CatalogPage","commitId":"45a09c71-e639-4ab0-81de-0ac51deef6ea","commitTimeStamp":"2015-02-01T07:55:34.5170706Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page10.json","@type":"CatalogPage","commitId":"3c9bc582-e047-47a5-a9ee-86a371d75f09","commitTimeStamp":"2015-02-01T08:06:59.4261022Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page11.json","@type":"CatalogPage","commitId":"9f4ff8fd-9548-4af7-838f-a7ecf5bfb556","commitTimeStamp":"2015-02-01T08:16:46.9598079Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page12.json","@type":"CatalogPage","commitId":"63b2fbe9-421d-4f11-90ed-b8a7f90867e9","commitTimeStamp":"2015-02-01T08:25:48.4826012Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page13.json","@type":"CatalogPage","commitId":"0349c096-be04-4029-8def-8951be2ce273","commitTimeStamp":"2015-02-01T08:34:32.1791916Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page14.json","@type":"CatalogPage","commitId":"5ac82009-c506-452b-9ec0-4a23dfd0a139","commitTimeStamp":"2015-02-01T08:45:48.746707Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page15.json","@type":"CatalogPage","commitId":"8c7373fc-7b60-4ff6-8f37-59b73b80fd18","commitTimeStamp":"2015-02-01T08:57:40.9795454Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page16.json","@type":"CatalogPage","commitId":"6d0b5e8a-74d8-405a-88af-7f1039c840e8","commitTimeStamp":"2015-02-01T09:09:07.2925344Z","count":538},{"@id":"https://api.nuget.org/v3/catalog0/page17.json","@type":"CatalogPage","commitId":"c62fd0a9-6e6b-449d-bdd1-dace8cec99e9","commitTimeStamp":"2015-02-01T09:19:49.2760414Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page18.json","@type":"CatalogPage","commitId":"7ca5e615-ee03-4910-9f0d-f08a927245e3","commitTimeStamp":"2015-02-01T09:29:14.1153342Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page19.json","@type":"CatalogPage","commitId":"cd0f1fd5-1223-4012-88c1-831113f3a169","commitTimeStamp":"2015-02-01T09:40:53.9605425Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page20.json","@type":"CatalogPage","commitId":"af25c51f-86e6-49ac-b2f3-9d4adace3d7e","commitTimeStamp":"2015-02-01T09:52:22.37016Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page21.json","@type":"CatalogPage","commitId":"208041aa-69d5-4337-a912-70018127a7aa","commitTimeStamp":"2015-02-01T10:03:44.2881321Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page22.json","@type":"CatalogPage","commitId":"6c666678-e587-4175-90c1-c4709cbc763e","commitTimeStamp":"2015-02-01T10:14:45.9340251Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page23.json","@type":"CatalogPage","commitId":"1d833774-98ae-42d0-bcc7-3853d124502c","commitTimeStamp":"2015-02-01T10:24:21.3347999Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page24.json","@type":"CatalogPage","commitId":"8822d5b8-4a13-4cef-b4be-7a92d13bfc6c","commitTimeStamp":"2015-02-01T10:33:58.5576357Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page25.json","@type":"CatalogPage","commitId":"61f3c9a8-2882-4a82-8a9c-852f661e3196","commitTimeStamp":"2015-02-01T10:47:12.2353585Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page26.json","@type":"CatalogPage","commitId":"00000000-0000-0000-0000-000000000000","commitTimeStamp":"2015-02-01T10:59:39.9767757Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page27.json","@type":"CatalogPage","commitId":"374719f9-9fcb-4a32-b6d5-1903b7429779","commitTimeStamp":"2015-02-01T11:11:41.9083539Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page28.json","@type":"CatalogPage","commitId":"00000000-0000-0000-0000-000000000000","commitTimeStamp":"2015-02-01T11:23:46.8866726Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page29.json","@type":"CatalogPage","commitId":"00000000-0000-0000-0000-000000000000","commitTimeStamp":"2015-02-01T11:36:32.0153363Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page30.json","@type":"CatalogPage","commitId":"00000000-0000-0000-0000-000000000000","commitTimeStamp":"2015-02-01T11:50:04.0835313Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page31.json","@type":"CatalogPage","commitId":"2368da8c-3cce-4047-ae87-b1d98796ad5c","commitTimeStamp":"2015-02-01T12:04:17.1858358Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page32.json","@type":"CatalogPage","commitId":"41aac087-19e0-437e-9351-9b00b25d908d","commitTimeStamp":"2015-02-01T12:17:37.875788Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page33.json","@type":"CatalogPage","commitId":"50ed5ede-350a-4a9b-ab32-75b7e47ee3ab","commitTimeStamp":"2015-02-01T12:29:24.2922228Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page34.json","@type":"CatalogPage","commitId":"1aa1a28c-02ef-4199-abc4-3a8e4c784bed","commitTimeStamp":"2015-02-01T12:42:19.7685797Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page35.json","@type":"CatalogPage","commitId":"713d0893-d2db-489d-85f6-66bb596ca47b","commitTimeStamp":"2015-02-01T12:55:14.5891123Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page36.json","@type":"CatalogPage","commitId":"706e235b-697b-47df-bfe3-426f404aae7d","commitTimeStamp":"2015-02-01T13:09:37.5912977Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page37.json","@type":"CatalogPage","commitId":"ff434d2b-65d7-4782-9647-bec32c09919e","commitTimeStamp":"2015-02-01T13:22:42.37873Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page38.json","@type":"CatalogPage","commitId":"87df4d1a-8ff0-4d14-ae8d-0552347432a1","commitTimeStamp":"2015-02-01T13:36:18.2070162Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page39.json","@type":"CatalogPage","commitId":"42a6223a-07e5-4a25-a6e9-2e12b1e2e207","commitTimeStamp":"2015-02-01T13:51:36.3122739Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page40.json","@type":"CatalogPage","commitId":"59306216-d615-4761-b988-a63b6c743b52","commitTimeStamp":"2015-02-01T14:09:28.4658122Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page41.json","@type":"CatalogPage","commitId":"b58d261c-8e0c-4975-8dcf-aac9f49ce62b","commitTimeStamp":"2015-02-01T14:24:13.0080426Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page42.json","@type":"CatalogPage","commitId":"65223762-fadb-4f3e-9274-75ddfcb83d57","commitTimeStamp":"2015-02-01T14:36:58.257618Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page43.json","@type":"CatalogPage","commitId":"64141e21-f84d-4de9-89a5-97b62c11c7cf","commitTimeStamp":"2015-02-01T14:49:42.0549582Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page44.json","@type":"CatalogPage","commitId":"3d9b2986-c6ff-4424-8bc3-ca1900936778","commitTimeStamp":"2015-02-01T14:59:57.7507026Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page45.json","@type":"CatalogPage","commitId":"36236c75-18e0-4c3e-a9af-8530a2b6c57f","commitTimeStamp":"2015-02-01T15:10:59.9949458Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page46.json","@type":"CatalogPage","commitId":"1fb47e91-0c15-4923-bbdb-f4409ce9711a","commitTimeStamp":"2015-02-01T15:22:00.3374688Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page47.json","@type":"CatalogPage","commitId":"3322b67f-3f69-478e-9748-9dccda26e402","commitTimeStamp":"2015-02-01T15:33:31.066306Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page48.json","@type":"CatalogPage","commitId":"2eccdd85-478f-4f94-91a9-fceea438ee50","commitTimeStamp":"2015-02-01T15:42:30.511083Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page49.json","@type":"CatalogPage","commitId":"2221d26b-d88c-48e3-b499-d3e6e7ae3e2b","commitTimeStamp":"2015-02-01T15:53:02.7064394Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page50.json","@type":"CatalogPage","commitId":"b1aafd12-fd2f-4c2b-90d5-e6fbeb8155d2","commitTimeStamp":"2015-02-01T16:02:27.4600574Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page51.json","@type":"CatalogPage","commitId":"4c6c96a0-943a-4690-8f7f-31d7911ade99","commitTimeStamp":"2015-02-01T16:12:15.0932073Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page52.json","@type":"CatalogPage","commitId":"81f13753-572e-49e9-b562-485490a1e411","commitTimeStamp":"2015-02-01T16:23:31.9565114Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page53.json","@type":"CatalogPage","commitId":"9f7ea3ba-2e7c-4f19-91b5-cb8b96e91722","commitTimeStamp":"2015-02-01T16:35:52.255371Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page54.json","@type":"CatalogPage","commitId":"1709d3d1-eafe-4f17-ab1d-1064b9429dda","commitTimeStamp":"2015-02-01T16:45:45.1969707Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page55.json","@type":"CatalogPage","commitId":"bc46f62d-be80-404b-a187-cbdfe36fd191","commitTimeStamp":"2015-02-01T16:56:12.5296174Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page56.json","@type":"CatalogPage","commitId":"0d1b243b-d89b-4c8d-a224-80668d011736","commitTimeStamp":"2015-02-01T17:04:49.5292388Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page57.json","@type":"CatalogPage","commitId":"c8fbe871-b4e6-45d7-a57d-997f08d39856","commitTimeStamp":"2015-02-01T17:14:59.4311843Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page58.json","@type":"CatalogPage","commitId":"1bd226b0-3e96-4cc0-8bfc-f5300e8971a7","commitTimeStamp":"2015-02-01T17:23:13.5134632Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page59.json","@type":"CatalogPage","commitId":"e6e57dac-bccd-4490-b568-93dd8e5ea88d","commitTimeStamp":"2015-02-01T17:32:13.4643225Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page60.json","@type":"CatalogPage","commitId":"6cc7f91b-39ee-4601-85a1-ca5089b33398","commitTimeStamp":"2015-02-01T17:42:40.4785527Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page61.json","@type":"CatalogPage","commitId":"e33f7f03-1139-447a-b1ea-7d4890891af4","commitTimeStamp":"2015-02-01T17:52:30.6040221Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page62.json","@type":"CatalogPage","commitId":"06556814-7bf9-4f6b-a135-8eb89a9a6d02","commitTimeStamp":"2015-02-01T18:03:33.4269221Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page63.json","@type":"CatalogPage","commitId":"d0aae0b5-e1db-4c4b-a70d-32fa5fe6b66b","commitTimeStamp":"2015-02-01T18:14:38.5468392Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page64.json","@type":"CatalogPage","commitId":"7f63aef4-c070-40f9-aeea-efe335b9a70b","commitTimeStamp":"2015-02-01T18:24:22.9539508Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page65.json","@type":"CatalogPage","commitId":"c6aec5da-de41-4ba9-8d66-d92535b837b9","commitTimeStamp":"2015-02-01T18:35:34.5126267Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page66.json","@type":"CatalogPage","commitId":"e1010281-13e6-45b5-96c8-fe14e5856a30","commitTimeStamp":"2015-02-01T18:45:23.8838705Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page67.json","@type":"CatalogPage","commitId":"ea2dec9d-6cdd-40d8-9eb1-53e661f2691e","commitTimeStamp":"2015-02-01T18:54:51.0719205Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page68.json","@type":"CatalogPage","commitId":"2c23cf67-042c-4856-b4c0-07c4ad1b3d02","commitTimeStamp":"2015-02-01T19:04:05.631379Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page69.json","@type":"CatalogPage","commitId":"8b4e129b-91af-4ccf-b2c8-01baaaadeb5a","commitTimeStamp":"2015-02-01T19:13:40.7380254Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page70.json","@type":"CatalogPage","commitId":"1f067c71-588c-4cb9-a0d5-396bcc1967f0","commitTimeStamp":"2015-02-01T19:23:23.0899995Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page71.json","@type":"CatalogPage","commitId":"de892af7-b8b5-41e5-b1f2-3586911ba2fa","commitTimeStamp":"2015-02-01T19:32:41.5655623Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page72.json","@type":"CatalogPage","commitId":"b111741c-3c76-4464-8498-af09ff8f2d94","commitTimeStamp":"2015-02-01T19:42:35.3657516Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page73.json","@type":"CatalogPage","commitId":"77a9fab4-1548-4a22-94fd-11ae24c54df6","commitTimeStamp":"2015-02-01T19:51:07.1240824Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page74.json","@type":"CatalogPage","commitId":"a0b60ec3-daf3-4bfe-ae4c-d71ef3011d18","commitTimeStamp":"2015-02-01T19:59:49.5855713Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page75.json","@type":"CatalogPage","commitId":"a37f2f5a-93e5-4d42-a992-cc574e6918a9","commitTimeStamp":"2015-02-01T20:09:44.1205988Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page76.json","@type":"CatalogPage","commitId":"74ddd0f8-87e8-49b0-9717-836df6c7094c","commitTimeStamp":"2015-02-01T20:18:47.7259607Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page77.json","@type":"CatalogPage","commitId":"5e53c415-62f2-4b66-862c-adbcd49a56e6","commitTimeStamp":"2015-02-01T20:29:39.9656468Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page78.json","@type":"CatalogPage","commitId":"17e02540-597b-4ac8-826c-5fc77a630a4d","commitTimeStamp":"2015-02-01T20:39:45.6941861Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page79.json","@type":"CatalogPage","commitId":"61c0a706-52a5-4600-a717-503f1972ce8b","commitTimeStamp":"2015-02-01T20:50:07.4771322Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page80.json","@type":"CatalogPage","commitId":"11a16607-7f9c-4535-b00d-c64e2f4e7902","commitTimeStamp":"2015-02-01T21:00:38.2511381Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page81.json","@type":"CatalogPage","commitId":"6d7fc22a-501e-4598-a2db-bc564a29d4a9","commitTimeStamp":"2015-02-01T21:10:11.1474898Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page82.json","@type":"CatalogPage","commitId":"283008a7-73e1-4c3f-9883-31841b6eb2cd","commitTimeStamp":"2015-02-01T21:20:16.4171015Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page83.json","@type":"CatalogPage","commitId":"4b11594c-1403-4600-b45f-dc1b5a438a7f","commitTimeStamp":"2015-02-01T21:30:43.1700766Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page84.json","@type":"CatalogPage","commitId":"16281542-79b5-4441-8f00-e7fac41631b8","commitTimeStamp":"2015-02-01T21:40:47.5178853Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page85.json","@type":"CatalogPage","commitId":"b6158ba6-d55a-46e2-8e79-30d471d7847d","commitTimeStamp":"2015-02-01T21:51:09.6196783Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page86.json","@type":"CatalogPage","commitId":"855a4fe6-426d-4343-bf06-a721c57c82dc","commitTimeStamp":"2015-02-01T22:00:51.7235773Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page87.json","@type":"CatalogPage","commitId":"ae7e3524-2688-4a28-9a4f-287d41378f39","commitTimeStamp":"2015-02-01T22:11:31.7570831Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page88.json","@type":"CatalogPage","commitId":"17b3632b-926c-4342-924d-59f45b633e4c","commitTimeStamp":"2015-02-01T22:21:14.6222101Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page89.json","@type":"CatalogPage","commitId":"84dcd0ee-9eb9-4d7d-a3de-da6fd366f0f4","commitTimeStamp":"2015-02-01T22:29:30.0345687Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page90.json","@type":"CatalogPage","commitId":"ec636145-ef1a-4c09-ac6f-d1d83b56c1ad","commitTimeStamp":"2015-02-01T22:38:57.2785352Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page91.json","@type":"CatalogPage","commitId":"4048c43f-12a1-4ba7-af3f-c2292c35f1a3","commitTimeStamp":"2015-02-01T22:47:56.2574265Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page92.json","@type":"CatalogPage","commitId":"1315e98b-abc7-4218-bd9a-39c0940576ef","commitTimeStamp":"2015-02-01T22:57:09.7321264Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page93.json","@type":"CatalogPage","commitId":"043906aa-a0bf-45e5-bb56-8dfcf7c6ca5c","commitTimeStamp":"2015-02-01T23:05:39.065951Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page94.json","@type":"CatalogPage","commitId":"b2923428-959a-4d4b-9099-d4cffe914c48","commitTimeStamp":"2015-02-01T23:15:57.3993091Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page95.json","@type":"CatalogPage","commitId":"5f9ade8b-342d-46e8-a416-681bb0648d40","commitTimeStamp":"2015-02-01T23:24:56.1168725Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page96.json","@type":"CatalogPage","commitId":"1746bdf7-cbcc-489c-a415-9db45094436f","commitTimeStamp":"2015-02-01T23:32:58.4511292Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page97.json","@type":"CatalogPage","commitId":"72a135ef-78ba-47c5-b008-0d122a0dd3aa","commitTimeStamp":"2015-02-01T23:42:52.7810612Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page98.json","@type":"CatalogPage","commitId":"119d7d49-bc66-4406-95e4-1d9ec6a2ef8f","commitTimeStamp":"2015-02-01T23:51:19.1471156Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page99.json","@type":"CatalogPage","commitId":"6fcaa8ff-e794-4a92-85b3-35ef1deb5104","commitTimeStamp":"2015-02-02T00:00:40.342449Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page100.json","@type":"CatalogPage","commitId":"ca75f2a3-a005-47e4-8b5e-ae6727a00f04","commitTimeStamp":"2015-02-02T00:10:51.5039017Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page101.json","@type":"CatalogPage","commitId":"e5cf32ce-64f7-4966-ae60-31107e51bdb3","commitTimeStamp":"2015-02-02T00:21:02.2260478Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page102.json","@type":"CatalogPage","commitId":"7d09c4fa-917f-40c9-b050-f61f22b9f4fe","commitTimeStamp":"2015-02-02T00:31:24.8050789Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page103.json","@type":"CatalogPage","commitId":"dcb5756f-fd31-4cd0-969b-7940774e85a5","commitTimeStamp":"2015-02-02T00:41:29.072463Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page104.json","@type":"CatalogPage","commitId":"2064d78c-f11c-47a8-b687-808d0764d74a","commitTimeStamp":"2015-02-02T00:51:18.4505417Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page105.json","@type":"CatalogPage","commitId":"83264e77-bcdc-4049-92fd-aceafbc43abe","commitTimeStamp":"2015-02-02T01:02:08.6696643Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page106.json","@type":"CatalogPage","commitId":"0cffa2ec-8128-402e-81af-456abc35de23","commitTimeStamp":"2015-02-02T01:12:55.5420214Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page107.json","@type":"CatalogPage","commitId":"f67af3da-f091-4548-8dcb-e05c88aae6ee","commitTimeStamp":"2015-02-02T01:23:16.8550747Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page108.json","@type":"CatalogPage","commitId":"b414d7f2-c6be-4315-a0e4-45723b7c83af","commitTimeStamp":"2015-02-02T01:32:47.351392Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page109.json","@type":"CatalogPage","commitId":"657b571c-7531-498d-962e-3841228aef5c","commitTimeStamp":"2015-02-02T01:42:57.2891812Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page110.json","@type":"CatalogPage","commitId":"c568635c-632c-4edd-8c6c-53921c5d32d6","commitTimeStamp":"2015-02-02T01:54:07.0809335Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page111.json","@type":"CatalogPage","commitId":"58927331-0cc3-435a-8f42-a6306cd9a36a","commitTimeStamp":"2015-02-02T02:05:22.5187283Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page112.json","@type":"CatalogPage","commitId":"0e0dee18-2bbe-45fd-bb86-c8bf8a754ed3","commitTimeStamp":"2015-02-02T02:18:16.3472108Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page113.json","@type":"CatalogPage","commitId":"a42f6b37-83be-4436-b499-681597f90056","commitTimeStamp":"2015-02-02T02:29:48.3167332Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page114.json","@type":"CatalogPage","commitId":"f2910867-d1a5-4fa5-b39f-84ae84758b99","commitTimeStamp":"2015-02-02T02:42:57.6161566Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page115.json","@type":"CatalogPage","commitId":"116d711a-c24a-4102-b00b-fff61e2a0bc5","commitTimeStamp":"2015-02-02T02:54:48.230787Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page116.json","@type":"CatalogPage","commitId":"faa83e4c-9012-452c-a72a-45384e7bad1e","commitTimeStamp":"2015-02-02T03:07:17.2821818Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page117.json","@type":"CatalogPage","commitId":"f4d66d52-0ff8-4649-acd4-0d345117cf4f","commitTimeStamp":"2015-02-02T03:17:52.7947969Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page118.json","@type":"CatalogPage","commitId":"1a33a8c4-32e3-4b50-af52-284ae17e59cc","commitTimeStamp":"2015-02-02T03:26:43.6220936Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page119.json","@type":"CatalogPage","commitId":"9b93e92d-576c-45ad-9a82-80fbc0608641","commitTimeStamp":"2015-02-02T03:36:12.35412Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page120.json","@type":"CatalogPage","commitId":"d7a21db8-2adb-414c-b111-011865ffe038","commitTimeStamp":"2015-02-02T03:43:54.5374317Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page121.json","@type":"CatalogPage","commitId":"34d017ca-9319-4dfb-9a15-9358bde13b1b","commitTimeStamp":"2015-02-02T03:51:41.5037451Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page122.json","@type":"CatalogPage","commitId":"370cc52b-9ea8-49b4-b939-039f72dd06cc","commitTimeStamp":"2015-02-02T04:01:50.6828993Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page123.json","@type":"CatalogPage","commitId":"d654bd07-01ab-4b88-8b1a-4a95f0a7b3c0","commitTimeStamp":"2015-02-02T04:13:15.5082495Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page124.json","@type":"CatalogPage","commitId":"d6fac537-9afe-4369-97d8-de08d196193d","commitTimeStamp":"2015-02-02T04:24:34.4209685Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page125.json","@type":"CatalogPage","commitId":"4cc856f0-febc-4ef2-b03d-e735d1beb6df","commitTimeStamp":"2015-02-02T04:36:35.2083545Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page126.json","@type":"CatalogPage","commitId":"d3a71288-929a-4256-b757-773357828530","commitTimeStamp":"2015-02-02T04:50:08.0056107Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page127.json","@type":"CatalogPage","commitId":"a0a2ed34-57c5-4362-81bb-ddadb33a0c8f","commitTimeStamp":"2015-02-02T16:43:56.1117359Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page128.json","@type":"CatalogPage","commitId":"5846192c-6c8a-4d0f-af2f-ccfbb4fd8c50","commitTimeStamp":"2015-02-02T16:53:24.9758074Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page129.json","@type":"CatalogPage","commitId":"8d9e07e5-b308-4bda-a6d8-b529e834eaae","commitTimeStamp":"2015-02-02T17:07:15.8805201Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page130.json","@type":"CatalogPage","commitId":"5e25a29a-2070-40b4-a1d9-ea30dbaa2831","commitTimeStamp":"2015-02-02T17:16:03.7513908Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page131.json","@type":"CatalogPage","commitId":"431bc3d8-60c2-4856-b4a6-dbb175554351","commitTimeStamp":"2015-02-02T17:26:15.6542828Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page132.json","@type":"CatalogPage","commitId":"13ec4bab-c679-42b3-830c-fc2d4de9afdc","commitTimeStamp":"2015-02-02T17:36:50.9851046Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page133.json","@type":"CatalogPage","commitId":"baa8a473-70e4-445b-b2d5-95e6db49de8b","commitTimeStamp":"2015-02-02T17:45:49.1036254Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page134.json","@type":"CatalogPage","commitId":"8c518cb5-376f-41c9-b766-ac28fea2a235","commitTimeStamp":"2015-02-02T17:56:04.7793769Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page135.json","@type":"CatalogPage","commitId":"bcfee23e-9289-4fce-bf4e-a9e01cce9b61","commitTimeStamp":"2015-02-02T18:07:08.3379531Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page136.json","@type":"CatalogPage","commitId":"1858b764-cf3f-4998-b9b3-e25b3a256dfd","commitTimeStamp":"2015-02-02T18:16:54.4080086Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page137.json","@type":"CatalogPage","commitId":"f88a2340-40da-4b40-8527-1a63dd406494","commitTimeStamp":"2015-02-02T18:26:00.8493824Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page138.json","@type":"CatalogPage","commitId":"223cf899-fc32-45ff-9e09-127c4327812d","commitTimeStamp":"2015-02-02T18:36:22.1386255Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page139.json","@type":"CatalogPage","commitId":"938f0f55-9224-4d62-866d-5398e6217645","commitTimeStamp":"2015-02-02T18:44:43.0932129Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page140.json","@type":"CatalogPage","commitId":"1f5a2261-d329-4b70-9777-ec316bcab055","commitTimeStamp":"2015-02-02T18:53:43.0599699Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page141.json","@type":"CatalogPage","commitId":"3767e61e-1861-41e3-b6eb-f91c400d6d45","commitTimeStamp":"2015-02-02T19:02:01.3754391Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page142.json","@type":"CatalogPage","commitId":"5844b0ea-3721-4016-a0ca-c1f33d6198b4","commitTimeStamp":"2015-02-02T19:10:44.8005001Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page143.json","@type":"CatalogPage","commitId":"9afd6e9f-ae64-446e-8042-2fd546b4a8f7","commitTimeStamp":"2015-02-02T19:19:37.6656141Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page144.json","@type":"CatalogPage","commitId":"0482f971-62cd-4a6f-8151-5e2b82f39c21","commitTimeStamp":"2015-02-02T19:27:51.4895423Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page145.json","@type":"CatalogPage","commitId":"3879d54c-38f4-4eca-b1c0-3117a97e02ab","commitTimeStamp":"2015-02-02T19:35:20.528202Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page146.json","@type":"CatalogPage","commitId":"b578f51d-3c0f-480a-89d7-04a089af665d","commitTimeStamp":"2015-02-02T19:44:33.6604813Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page147.json","@type":"CatalogPage","commitId":"79c10a68-00e8-4fad-8db5-33448db342da","commitTimeStamp":"2015-02-02T19:54:42.2911539Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page148.json","@type":"CatalogPage","commitId":"a5655318-5006-470a-bdca-1bd4703642b2","commitTimeStamp":"2015-02-02T20:04:40.4909177Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page149.json","@type":"CatalogPage","commitId":"1c018d65-32e5-4623-b025-e3b3348c606e","commitTimeStamp":"2015-02-02T20:16:43.5229248Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page150.json","@type":"CatalogPage","commitId":"c1305c01-8779-42d4-ac4e-cbfee5ea760a","commitTimeStamp":"2015-02-02T20:27:27.1210733Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page151.json","@type":"CatalogPage","commitId":"d88ae05c-9828-424d-a9d9-a8b757095a5c","commitTimeStamp":"2015-02-02T20:35:42.2589027Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page152.json","@type":"CatalogPage","commitId":"9b44ae04-71dc-42bf-93fc-17c2605e23f5","commitTimeStamp":"2015-02-02T20:43:15.3842289Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page153.json","@type":"CatalogPage","commitId":"102f59eb-a41f-4d69-b6be-efe4b5100c8c","commitTimeStamp":"2015-02-02T20:52:18.3440248Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page154.json","@type":"CatalogPage","commitId":"e80128bd-0b31-4c60-8fdd-8959ab938a97","commitTimeStamp":"2015-02-02T21:00:59.3835343Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page155.json","@type":"CatalogPage","commitId":"12224037-d7fd-4cc0-b0c6-d399fad776d8","commitTimeStamp":"2015-02-02T21:10:47.8737446Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page156.json","@type":"CatalogPage","commitId":"3bafdf91-b73d-4203-9d31-7cbe00b7dd89","commitTimeStamp":"2015-02-02T21:20:34.2329147Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page157.json","@type":"CatalogPage","commitId":"eee04a1b-51c1-4e65-84e8-1254b61ffb36","commitTimeStamp":"2015-02-02T21:30:35.9861255Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page158.json","@type":"CatalogPage","commitId":"42e3a5c7-b5d9-424a-ae57-c5ccdb151d8c","commitTimeStamp":"2015-02-02T21:40:11.8700867Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page159.json","@type":"CatalogPage","commitId":"12158bc2-ffc6-43ae-b6e1-dbb7cbd17ba5","commitTimeStamp":"2015-02-02T21:51:49.8659999Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page160.json","@type":"CatalogPage","commitId":"6a3edd1e-1355-4022-8ab8-905ab5b84272","commitTimeStamp":"2015-02-02T21:59:49.1307543Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page161.json","@type":"CatalogPage","commitId":"d7ebde2a-4b48-4628-a428-d38607c9896b","commitTimeStamp":"2015-02-02T22:13:49.5829366Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page162.json","@type":"CatalogPage","commitId":"ce65b4b5-6743-4086-947a-deb48fa27fea","commitTimeStamp":"2015-02-02T22:24:00.6563125Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page163.json","@type":"CatalogPage","commitId":"c8832af5-1871-422f-a4c6-28727ad9ca20","commitTimeStamp":"2015-02-02T22:32:42.388769Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page164.json","@type":"CatalogPage","commitId":"91c8ba76-4c55-44f6-bb64-c184d9107948","commitTimeStamp":"2015-02-02T22:41:56.8528914Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page165.json","@type":"CatalogPage","commitId":"51cc10ba-7b59-45d1-8a4d-17f1f79012fe","commitTimeStamp":"2015-02-02T22:51:05.0414231Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page166.json","@type":"CatalogPage","commitId":"e2ea4b3c-3af6-4a1c-9e98-dce8f3ba28b9","commitTimeStamp":"2015-02-02T23:00:41.02623Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page167.json","@type":"CatalogPage","commitId":"f92c0654-d1d1-4f12-abbd-579dbd9f239f","commitTimeStamp":"2015-02-02T23:09:45.162409Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page168.json","@type":"CatalogPage","commitId":"6ec0cdaf-02a8-4272-8779-696a78c21dcd","commitTimeStamp":"2015-02-02T23:17:09.3481109Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page169.json","@type":"CatalogPage","commitId":"58e8c34b-112e-4316-b27e-207c04c7589f","commitTimeStamp":"2015-02-02T23:28:10.5310009Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page170.json","@type":"CatalogPage","commitId":"f1367183-2eec-4319-bc83-1eaf9514ab88","commitTimeStamp":"2015-02-02T23:36:49.1274998Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page171.json","@type":"CatalogPage","commitId":"4ce3005d-64e1-4a38-8e00-babd935d7f04","commitTimeStamp":"2015-02-02T23:45:10.9882852Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page172.json","@type":"CatalogPage","commitId":"06796786-fb83-45bf-b9b6-cc4b6e9769f7","commitTimeStamp":"2015-02-02T23:53:36.0544957Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page173.json","@type":"CatalogPage","commitId":"8f55cf5a-91df-4b81-808d-6a4483011fef","commitTimeStamp":"2015-02-03T00:04:50.5949079Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page174.json","@type":"CatalogPage","commitId":"55ba1977-97f1-4dea-ad4e-e2fc9087e66c","commitTimeStamp":"2015-02-03T00:13:39.6120445Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page175.json","@type":"CatalogPage","commitId":"3e7eb703-4cb6-439d-8bed-f052f3c2be94","commitTimeStamp":"2015-02-03T00:23:17.6347663Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page176.json","@type":"CatalogPage","commitId":"1a20b7b3-2f00-4b49-8abf-23bc93e17001","commitTimeStamp":"2015-02-03T00:35:10.6407035Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page177.json","@type":"CatalogPage","commitId":"88b9beeb-2e42-4294-82f0-d7b1af618367","commitTimeStamp":"2015-02-03T00:46:53.6699778Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page178.json","@type":"CatalogPage","commitId":"181ded66-45b9-428d-8538-09242a3f1c39","commitTimeStamp":"2015-02-03T01:02:26.2717626Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page179.json","@type":"CatalogPage","commitId":"9f1a235a-d5e0-4d2c-b74e-8fd05d056be5","commitTimeStamp":"2015-02-03T01:12:21.3965086Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page180.json","@type":"CatalogPage","commitId":"e41f3df0-6e2b-4b5f-9032-12a4e6868f8e","commitTimeStamp":"2015-02-03T01:22:11.4393964Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page181.json","@type":"CatalogPage","commitId":"bf9854f6-358d-4a55-9786-6e06dc2d8e21","commitTimeStamp":"2015-02-03T01:30:21.6847056Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page182.json","@type":"CatalogPage","commitId":"764208be-50a1-45dc-9d70-a3c04f52e08d","commitTimeStamp":"2015-02-03T01:39:48.8142731Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page183.json","@type":"CatalogPage","commitId":"5b3cfb3e-82a4-46dd-a736-c1824c962683","commitTimeStamp":"2015-02-03T01:48:40.7684116Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page184.json","@type":"CatalogPage","commitId":"cc7219b3-2e68-4b58-8f64-9c6895055e7d","commitTimeStamp":"2015-02-03T01:59:17.6397178Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page185.json","@type":"CatalogPage","commitId":"5599af1f-d25c-4818-b3c5-e0a5dbca65f7","commitTimeStamp":"2015-02-03T02:08:58.8535547Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page186.json","@type":"CatalogPage","commitId":"703481b9-8a3a-4f5c-a6fd-08e1bc6feee3","commitTimeStamp":"2015-02-03T02:19:13.3933306Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page187.json","@type":"CatalogPage","commitId":"d657319f-9f5b-49b3-bf0f-832689dc25f9","commitTimeStamp":"2015-02-03T02:28:27.7747878Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page188.json","@type":"CatalogPage","commitId":"e5fd1bb5-bf24-4554-9908-7769f4939562","commitTimeStamp":"2015-02-03T02:38:04.9155207Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page189.json","@type":"CatalogPage","commitId":"66443818-6955-4edf-afbf-f7da2d08637f","commitTimeStamp":"2015-02-03T02:46:24.5435606Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page190.json","@type":"CatalogPage","commitId":"7dffad46-a9a6-4b87-b311-b74e88da0695","commitTimeStamp":"2015-02-03T02:55:40.7514038Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page191.json","@type":"CatalogPage","commitId":"d49a8701-694f-493c-bf71-8100419e9b1f","commitTimeStamp":"2015-02-03T03:06:34.8130413Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page192.json","@type":"CatalogPage","commitId":"7ef8a91c-dc45-4d10-8df3-f0961e12be14","commitTimeStamp":"2015-02-03T03:16:13.1612615Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page193.json","@type":"CatalogPage","commitId":"bea0926b-e7e7-42c5-97c2-fa57b30d29a2","commitTimeStamp":"2015-02-03T03:24:48.2342436Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page194.json","@type":"CatalogPage","commitId":"fb31e73b-c2fc-4846-8be0-52f923f16496","commitTimeStamp":"2015-02-03T03:32:27.0410848Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page195.json","@type":"CatalogPage","commitId":"53606a9b-14ca-4090-b34f-199b4cd4dc9f","commitTimeStamp":"2015-02-03T03:42:38.2560514Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page196.json","@type":"CatalogPage","commitId":"eff8a869-89fe-448a-8e73-5d1d76115dca","commitTimeStamp":"2015-02-03T03:51:23.7289678Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page197.json","@type":"CatalogPage","commitId":"95c91351-93a0-4e1c-a46a-0bcaaa2edbfc","commitTimeStamp":"2015-02-03T03:59:42.7227488Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page198.json","@type":"CatalogPage","commitId":"8d9f7638-fae2-486f-8320-dca751e54215","commitTimeStamp":"2015-02-03T04:09:27.7904852Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page199.json","@type":"CatalogPage","commitId":"fe255dfb-8c77-4d5c-851e-50151731567f","commitTimeStamp":"2015-02-03T04:22:11.3453757Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page200.json","@type":"CatalogPage","commitId":"ccaf966c-8242-48d7-8aad-2d1ade7281cf","commitTimeStamp":"2015-02-03T04:30:28.403543Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page201.json","@type":"CatalogPage","commitId":"4d13b798-1538-4db8-9828-5b53518ad646","commitTimeStamp":"2015-02-03T04:39:07.8205419Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page202.json","@type":"CatalogPage","commitId":"a32c668f-a4f3-4f06-9e96-b2007b83d2b7","commitTimeStamp":"2015-02-03T04:47:25.9615778Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page203.json","@type":"CatalogPage","commitId":"b86d1fa7-5334-45f3-949f-03ecb62f8952","commitTimeStamp":"2015-02-03T04:57:20.4152447Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page204.json","@type":"CatalogPage","commitId":"5da7bbca-6aa7-4ae3-af03-674f8803f328","commitTimeStamp":"2015-02-03T05:24:30.7692719Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page205.json","@type":"CatalogPage","commitId":"8b43973f-e500-4b67-b2bd-004a55ab4da1","commitTimeStamp":"2015-02-03T05:31:12.2086202Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page206.json","@type":"CatalogPage","commitId":"ece723d0-027d-49ca-a3c2-76667b1cea4b","commitTimeStamp":"2015-02-03T05:39:06.0993701Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page207.json","@type":"CatalogPage","commitId":"daed3478-6f79-468c-9df3-76fdbce6f029","commitTimeStamp":"2015-02-03T05:45:57.0869507Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page208.json","@type":"CatalogPage","commitId":"558a5a2c-68c7-43e7-a325-77bad1d2cbe2","commitTimeStamp":"2015-02-03T05:53:28.3121656Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page209.json","@type":"CatalogPage","commitId":"2027586a-1101-407e-8931-84fc980f1f7b","commitTimeStamp":"2015-02-03T06:03:17.2532343Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page210.json","@type":"CatalogPage","commitId":"9a26e1ea-d14c-44ea-84d9-3ae7f1bcb47d","commitTimeStamp":"2015-02-03T06:16:23.8541799Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page211.json","@type":"CatalogPage","commitId":"8a3b24ae-34ef-4fbf-a818-b06a9c987cd7","commitTimeStamp":"2015-02-03T06:24:54.4917731Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page212.json","@type":"CatalogPage","commitId":"909445b0-4396-4a63-b296-aa668202ebed","commitTimeStamp":"2015-02-03T06:34:17.4400481Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page213.json","@type":"CatalogPage","commitId":"702c0ff6-881b-479f-8ed1-58e0e9326ab2","commitTimeStamp":"2015-02-03T06:42:35.1148918Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page214.json","@type":"CatalogPage","commitId":"1382e48f-dcb5-426c-83d5-f95f2a92d868","commitTimeStamp":"2015-02-03T06:50:38.1169518Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page215.json","@type":"CatalogPage","commitId":"c2491c78-21e4-475a-ba29-b35d09de60a3","commitTimeStamp":"2015-02-03T06:57:50.4640722Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page216.json","@type":"CatalogPage","commitId":"7f25180a-5977-4d73-8ea8-2129013c36e2","commitTimeStamp":"2015-02-03T07:04:49.9528154Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page217.json","@type":"CatalogPage","commitId":"82172756-24a5-456e-a8fd-4cb456dbc912","commitTimeStamp":"2015-02-03T07:12:17.3330518Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page218.json","@type":"CatalogPage","commitId":"367ca34a-6be9-4343-b923-e1806b04c268","commitTimeStamp":"2015-02-03T07:19:49.5891235Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page219.json","@type":"CatalogPage","commitId":"6f5a5525-f26a-477b-b56b-ee1fb3d07754","commitTimeStamp":"2015-02-03T07:29:08.8665326Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page220.json","@type":"CatalogPage","commitId":"27e3d47a-5c24-4043-9f56-5e3786326880","commitTimeStamp":"2015-02-03T07:35:50.2955818Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page221.json","@type":"CatalogPage","commitId":"9749b9f6-26d2-462b-a329-552eaf020813","commitTimeStamp":"2015-02-03T07:43:09.3490772Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page222.json","@type":"CatalogPage","commitId":"66f563db-bd09-45aa-aa0a-eb707dbeda37","commitTimeStamp":"2015-02-03T07:50:15.8872173Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page223.json","@type":"CatalogPage","commitId":"a8d3bd25-9a4e-4f6d-beff-1c5b563d54c3","commitTimeStamp":"2015-02-03T07:56:41.9851017Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page224.json","@type":"CatalogPage","commitId":"82c992e3-3ae1-4a2f-af21-47fd41a3927d","commitTimeStamp":"2015-02-03T08:05:21.3674197Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page225.json","@type":"CatalogPage","commitId":"8c4eba57-3e32-4355-af64-3cb84923165b","commitTimeStamp":"2015-02-03T08:15:18.5961574Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page226.json","@type":"CatalogPage","commitId":"e3535e93-b8c3-4988-bccc-5c6843cbfec5","commitTimeStamp":"2015-02-03T08:23:41.9039624Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page227.json","@type":"CatalogPage","commitId":"1e47a1ac-a54e-4017-9399-58819cf7a454","commitTimeStamp":"2015-02-03T08:31:48.2319101Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page228.json","@type":"CatalogPage","commitId":"5730b1f6-b51f-4997-adb5-4d07888d4cd7","commitTimeStamp":"2015-02-03T08:39:02.1945654Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page229.json","@type":"CatalogPage","commitId":"44a01ad0-d7c3-47e5-9edb-6e4450e77d50","commitTimeStamp":"2015-02-03T08:47:26.532934Z","count":538},{"@id":"https://api.nuget.org/v3/catalog0/page230.json","@type":"CatalogPage","commitId":"4f915725-0ab1-4f38-9522-2b4cb3c97b87","commitTimeStamp":"2015-02-03T08:54:15.5623182Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page231.json","@type":"CatalogPage","commitId":"603fbf2c-99c3-44b7-a9ef-0525a9c593ee","commitTimeStamp":"2015-02-03T09:01:59.6727901Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page232.json","@type":"CatalogPage","commitId":"33ce3a8f-67d9-486e-a5ac-e949d8e0c30d","commitTimeStamp":"2015-02-03T09:09:38.8054624Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page233.json","@type":"CatalogPage","commitId":"399f1c83-8cc6-49f1-87db-f455b5c91f46","commitTimeStamp":"2015-02-03T09:16:35.8691333Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page234.json","@type":"CatalogPage","commitId":"14bacaa4-2c51-4c68-89bd-878ebcb9de9f","commitTimeStamp":"2015-02-03T09:23:41.1198307Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page235.json","@type":"CatalogPage","commitId":"ccc6ea3d-0a47-44d7-9be9-d093422c20aa","commitTimeStamp":"2015-02-03T09:31:25.4338767Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page236.json","@type":"CatalogPage","commitId":"ab7cc4b0-3176-4cbf-a7bc-f9e3d4586ca7","commitTimeStamp":"2015-02-03T09:38:29.3926071Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page237.json","@type":"CatalogPage","commitId":"a353ae72-ced8-4a7b-a2dd-40ddfd32dd68","commitTimeStamp":"2015-02-03T09:45:40.5093109Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page238.json","@type":"CatalogPage","commitId":"1388ea94-9916-42ec-a647-02825095653f","commitTimeStamp":"2015-02-03T09:53:08.4537487Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page239.json","@type":"CatalogPage","commitId":"70170f1b-0403-4faa-a291-26a3123192cb","commitTimeStamp":"2015-02-03T10:00:50.5978541Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page240.json","@type":"CatalogPage","commitId":"c5ae7c1e-7d2b-4c16-b5dd-b3fd5038a257","commitTimeStamp":"2015-02-03T10:07:25.5534367Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page241.json","@type":"CatalogPage","commitId":"04d19af0-5f17-4028-9bee-ef47ef73390e","commitTimeStamp":"2015-02-03T10:13:12.9927635Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page242.json","@type":"CatalogPage","commitId":"7983182d-760f-4a34-b270-afa53b4be1b0","commitTimeStamp":"2015-02-03T10:19:19.2269424Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page243.json","@type":"CatalogPage","commitId":"19bdcaa9-81bd-4a54-96d4-f84a1ae357e3","commitTimeStamp":"2015-02-03T10:25:24.883194Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page244.json","@type":"CatalogPage","commitId":"b5c46527-1c4d-4c87-9d95-edc7056faff8","commitTimeStamp":"2015-02-03T10:32:21.9390672Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page245.json","@type":"CatalogPage","commitId":"5fc9fa04-9292-4740-af7f-943b0af4527a","commitTimeStamp":"2015-02-03T10:38:52.7011604Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page246.json","@type":"CatalogPage","commitId":"0bad0e84-1fdc-4d2e-af4f-9ef726857326","commitTimeStamp":"2015-02-03T10:48:07.6288281Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page247.json","@type":"CatalogPage","commitId":"8eb476d1-1d29-478d-9530-993563a90bbd","commitTimeStamp":"2015-02-03T10:56:39.2112101Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page248.json","@type":"CatalogPage","commitId":"327d5a4a-915f-48db-8af3-03f34d8055a4","commitTimeStamp":"2015-02-03T11:04:07.4055416Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page249.json","@type":"CatalogPage","commitId":"b1f987c7-f4b3-4c39-8aee-21842eb22f52","commitTimeStamp":"2015-02-03T11:13:03.7880633Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page250.json","@type":"CatalogPage","commitId":"773d9e7d-31eb-4dfc-aefa-0983db126331","commitTimeStamp":"2015-02-03T11:21:37.2307187Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page251.json","@type":"CatalogPage","commitId":"5e21edee-6495-4216-a7fe-a988b176ebd4","commitTimeStamp":"2015-02-03T11:33:14.6286497Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page252.json","@type":"CatalogPage","commitId":"1282040f-8805-4108-b602-60b3d4d27d6c","commitTimeStamp":"2015-02-03T11:41:01.1059561Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page253.json","@type":"CatalogPage","commitId":"6269f03a-85ba-4333-a196-82792b5d2680","commitTimeStamp":"2015-02-03T11:48:32.3952553Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page254.json","@type":"CatalogPage","commitId":"81cbdbca-8913-4d31-9f4b-4a7512324282","commitTimeStamp":"2015-02-03T11:56:19.4944804Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page255.json","@type":"CatalogPage","commitId":"a4f45b48-b324-44c0-9db9-a049e32b59b0","commitTimeStamp":"2015-02-03T12:03:49.5497172Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page256.json","@type":"CatalogPage","commitId":"ffc74dae-822c-4517-8bf9-d3ac017a68c0","commitTimeStamp":"2015-02-03T12:13:47.1340454Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page257.json","@type":"CatalogPage","commitId":"5a3d2b6b-14eb-4339-9992-35958162425b","commitTimeStamp":"2015-02-03T12:22:10.1323722Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page258.json","@type":"CatalogPage","commitId":"740d1992-a29b-4f20-937b-6bf2e84596f9","commitTimeStamp":"2015-02-03T12:31:57.4746418Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page259.json","@type":"CatalogPage","commitId":"6151fbeb-c1a4-4467-97ea-30554f92b440","commitTimeStamp":"2015-02-03T12:39:41.7976908Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page260.json","@type":"CatalogPage","commitId":"96cf4418-b7d5-46c1-b511-b275cbc3edba","commitTimeStamp":"2015-02-03T12:45:14.5644248Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page261.json","@type":"CatalogPage","commitId":"df3f1113-f4ca-42fc-9fb5-b608e69ff436","commitTimeStamp":"2015-02-03T12:51:45.423751Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page262.json","@type":"CatalogPage","commitId":"076b7a95-f337-46fb-9a1f-114213523b50","commitTimeStamp":"2015-02-03T12:58:38.169212Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page263.json","@type":"CatalogPage","commitId":"10fb64c6-8e01-45d2-9ed0-6a75766dd473","commitTimeStamp":"2015-02-03T13:06:08.230788Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page264.json","@type":"CatalogPage","commitId":"55a6b991-79b2-4085-97cf-85a92743d13e","commitTimeStamp":"2015-02-03T13:13:48.2892426Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page265.json","@type":"CatalogPage","commitId":"0462d0e4-3c1d-4413-9183-d483ccf869de","commitTimeStamp":"2015-02-03T13:24:29.3952913Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page266.json","@type":"CatalogPage","commitId":"456ce389-0409-4d69-a2ba-edc8c098443f","commitTimeStamp":"2015-02-03T13:32:47.6123873Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page267.json","@type":"CatalogPage","commitId":"4be272a6-8a39-44a3-b2eb-a31c44fb0846","commitTimeStamp":"2015-02-03T13:41:11.2304921Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page268.json","@type":"CatalogPage","commitId":"b1f266b4-5d55-4c2a-82d9-41214ac08fb1","commitTimeStamp":"2015-02-03T13:48:59.6289184Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page269.json","@type":"CatalogPage","commitId":"60535ec9-123b-4b58-bc45-695e3fab85c3","commitTimeStamp":"2015-02-03T13:59:19.1126346Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page270.json","@type":"CatalogPage","commitId":"8bfcbff7-aece-4a6a-b8b9-283c82fee68d","commitTimeStamp":"2015-02-03T14:09:42.2923144Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page271.json","@type":"CatalogPage","commitId":"727c0737-4557-4eaa-aaf9-c6362acf83d6","commitTimeStamp":"2015-02-03T14:19:50.9073863Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page272.json","@type":"CatalogPage","commitId":"9f207c34-be15-4e5d-b730-c3a736a30659","commitTimeStamp":"2015-02-03T14:29:46.3161974Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page273.json","@type":"CatalogPage","commitId":"2a993ce4-b537-4a90-a2e6-f18a05c98242","commitTimeStamp":"2015-02-03T14:39:09.6158973Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page274.json","@type":"CatalogPage","commitId":"da76298f-a835-4387-88c0-56a91cd2aaf0","commitTimeStamp":"2015-02-03T14:47:28.5429712Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page275.json","@type":"CatalogPage","commitId":"17d1d5ef-f756-40ca-a5b3-00739a9fb047","commitTimeStamp":"2015-02-03T14:58:29.9132601Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page276.json","@type":"CatalogPage","commitId":"6e52ab1d-b81a-45a3-8930-d2b3f7bc43e2","commitTimeStamp":"2015-02-03T15:06:34.4167927Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page277.json","@type":"CatalogPage","commitId":"a00ffe3b-83f1-4a67-868f-2a6084f2c896","commitTimeStamp":"2015-02-03T15:13:34.9428258Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page278.json","@type":"CatalogPage","commitId":"bde217b8-2032-473f-a5e0-d1af2817e595","commitTimeStamp":"2015-02-03T15:21:34.9623808Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page279.json","@type":"CatalogPage","commitId":"ee3a4250-206f-42fe-89b8-f678c4579c6d","commitTimeStamp":"2015-02-03T15:29:06.0107154Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page280.json","@type":"CatalogPage","commitId":"da37001c-9ef1-4c5f-a4b8-19d3d600ac3c","commitTimeStamp":"2015-02-03T15:37:03.7774751Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page281.json","@type":"CatalogPage","commitId":"683dd504-936a-440f-bcac-9c7437903368","commitTimeStamp":"2015-02-03T15:44:39.2525023Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page282.json","@type":"CatalogPage","commitId":"6f441b15-6cc2-4f51-9c63-124f29c62363","commitTimeStamp":"2015-02-03T15:52:12.6522181Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page283.json","@type":"CatalogPage","commitId":"c5800fba-5190-4d69-aef4-4bffd0888885","commitTimeStamp":"2015-02-03T15:59:50.8633473Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page284.json","@type":"CatalogPage","commitId":"50f9f900-e0c1-4b0c-960d-557de46f2618","commitTimeStamp":"2015-02-03T16:07:53.933809Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page285.json","@type":"CatalogPage","commitId":"45d30008-ed1c-42cd-830d-a6daa1ac7d95","commitTimeStamp":"2015-02-03T16:16:44.9451285Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page286.json","@type":"CatalogPage","commitId":"59696702-b697-4b35-87f7-4595a4eee721","commitTimeStamp":"2015-02-03T16:24:39.1465107Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page287.json","@type":"CatalogPage","commitId":"b2f8cbe5-43f3-4bd9-b2ec-7269e799290b","commitTimeStamp":"2015-02-03T16:33:54.6183271Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page288.json","@type":"CatalogPage","commitId":"cb4b3889-7aea-4848-9c5d-39d84c618d5e","commitTimeStamp":"2015-02-03T16:43:16.2623263Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page289.json","@type":"CatalogPage","commitId":"8aed4cf7-9029-4540-8b2f-3d6040dd1efe","commitTimeStamp":"2015-02-03T16:52:38.0527808Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page290.json","@type":"CatalogPage","commitId":"8bf8eaae-387c-4e52-b1b3-0efe0d7b0aef","commitTimeStamp":"2015-02-03T17:00:39.9818599Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page291.json","@type":"CatalogPage","commitId":"14efcdbf-7fca-40dd-b8a9-1aefab9b8917","commitTimeStamp":"2015-02-03T17:07:43.5712863Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page292.json","@type":"CatalogPage","commitId":"0f78301e-9321-4e94-800b-ac7701c5a995","commitTimeStamp":"2015-02-03T17:16:36.4982708Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page293.json","@type":"CatalogPage","commitId":"3d1df078-6882-40f9-8093-8faa472c92d1","commitTimeStamp":"2015-02-03T17:24:14.0465273Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page294.json","@type":"CatalogPage","commitId":"d78e5e8a-6e2e-45a0-8867-ac0b84a87f24","commitTimeStamp":"2015-02-03T17:31:40.877606Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page295.json","@type":"CatalogPage","commitId":"9c0e8a15-2353-4c72-a470-fe788b64ffd5","commitTimeStamp":"2015-02-03T17:41:55.1469527Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page296.json","@type":"CatalogPage","commitId":"f20303c4-3843-4c83-b6b4-ed7c577562f9","commitTimeStamp":"2015-02-03T17:50:00.8618313Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page297.json","@type":"CatalogPage","commitId":"166a6a78-ada7-49ca-92c1-3f9ed7a0a235","commitTimeStamp":"2015-02-03T17:57:39.3498032Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page298.json","@type":"CatalogPage","commitId":"58a95c62-6b3c-4565-9aef-436c32e44022","commitTimeStamp":"2015-02-03T18:05:36.9921286Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page299.json","@type":"CatalogPage","commitId":"33bb0ca4-1066-42e6-af9c-f9e356fea546","commitTimeStamp":"2015-02-03T18:11:55.4593595Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page300.json","@type":"CatalogPage","commitId":"c6129b83-7cff-43a5-8344-21bacff57842","commitTimeStamp":"2015-02-03T18:20:14.4886425Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page301.json","@type":"CatalogPage","commitId":"4774a3bf-4cdd-4d5d-848e-5e638d0db70a","commitTimeStamp":"2015-02-03T18:30:41.8927408Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page302.json","@type":"CatalogPage","commitId":"aab1f692-4cd6-4ec8-8edf-388c06e15f97","commitTimeStamp":"2015-02-03T18:39:48.7855959Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page303.json","@type":"CatalogPage","commitId":"ccd7da8c-5bb5-4694-9928-2c10508836f6","commitTimeStamp":"2015-02-03T18:47:19.7433551Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page304.json","@type":"CatalogPage","commitId":"629339a5-dc62-4f54-b844-8f69d39eab30","commitTimeStamp":"2015-02-03T18:54:16.6868002Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page305.json","@type":"CatalogPage","commitId":"1c840968-6795-44c0-9dc7-0f738856644b","commitTimeStamp":"2015-02-03T19:01:59.6937319Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page306.json","@type":"CatalogPage","commitId":"b83da551-220f-4112-bd13-dec39d635afb","commitTimeStamp":"2015-02-03T19:10:51.7754838Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page307.json","@type":"CatalogPage","commitId":"45f8fd83-6466-4e05-bef4-28705faa25d6","commitTimeStamp":"2015-02-03T19:20:11.8557548Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page308.json","@type":"CatalogPage","commitId":"8dd487a3-491f-4a8d-9493-57d39d92503e","commitTimeStamp":"2015-02-03T19:29:54.0767801Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page309.json","@type":"CatalogPage","commitId":"f22f4380-951f-4644-af0d-5cd5cafa17df","commitTimeStamp":"2015-02-03T19:37:46.6967494Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page310.json","@type":"CatalogPage","commitId":"54d36346-971f-474e-ad25-a117851bdaa0","commitTimeStamp":"2015-02-03T19:47:51.4243329Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page311.json","@type":"CatalogPage","commitId":"ef1204e8-6199-49ee-aeec-ca41112c4094","commitTimeStamp":"2015-02-03T19:56:02.1343165Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page312.json","@type":"CatalogPage","commitId":"fd0daeae-efcc-4c94-b44f-59e32f330447","commitTimeStamp":"2015-02-03T20:04:34.9651303Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page313.json","@type":"CatalogPage","commitId":"12888955-19f1-4557-a6bd-b1a403ab922e","commitTimeStamp":"2015-02-03T20:13:03.4333836Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page314.json","@type":"CatalogPage","commitId":"81b9afbd-beca-4278-871a-80e9ea5b6b11","commitTimeStamp":"2015-02-03T20:20:33.3851725Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page315.json","@type":"CatalogPage","commitId":"07539d22-1ea3-4383-bf86-1273695b2ee3","commitTimeStamp":"2015-02-03T20:30:28.9746167Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page316.json","@type":"CatalogPage","commitId":"e3233f99-882a-4243-ab9c-9c7e07382263","commitTimeStamp":"2015-02-03T20:39:55.7481553Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page317.json","@type":"CatalogPage","commitId":"ba147257-7c36-4d15-ba16-ff7459ef1626","commitTimeStamp":"2015-02-03T20:47:27.8270269Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page318.json","@type":"CatalogPage","commitId":"8ef0c43b-51c0-41e9-a35a-0a8553cc448a","commitTimeStamp":"2015-02-03T20:55:25.2473999Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page319.json","@type":"CatalogPage","commitId":"b504419e-9abc-4145-a670-05d882c3dd3c","commitTimeStamp":"2015-02-03T21:02:14.3891879Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page320.json","@type":"CatalogPage","commitId":"2794a014-86f0-47a0-b6f7-ddf91da130b9","commitTimeStamp":"2015-02-03T21:10:48.541962Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page321.json","@type":"CatalogPage","commitId":"90c90b79-310e-4e01-975f-bfb7dc02e111","commitTimeStamp":"2015-02-03T21:20:36.5532806Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page322.json","@type":"CatalogPage","commitId":"65c942a5-bc9a-4f60-980e-8eefeb3f8362","commitTimeStamp":"2015-02-03T21:29:02.4461589Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page323.json","@type":"CatalogPage","commitId":"5743e06a-a918-44ac-9376-2cc21f65916b","commitTimeStamp":"2015-02-03T21:35:51.3776776Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page324.json","@type":"CatalogPage","commitId":"73491952-8c54-499f-8cf8-f03854c0192b","commitTimeStamp":"2015-02-03T21:43:21.1832896Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page325.json","@type":"CatalogPage","commitId":"01e48e8c-611f-4564-9ffe-728cf0e367d6","commitTimeStamp":"2015-02-03T21:49:42.0245591Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page326.json","@type":"CatalogPage","commitId":"ff1d4c07-f370-4fa9-a13a-32967325ccbc","commitTimeStamp":"2015-02-03T21:57:37.6739494Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page327.json","@type":"CatalogPage","commitId":"8715f766-d789-4138-b4d3-82116f76f724","commitTimeStamp":"2015-02-03T22:07:20.8152057Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page328.json","@type":"CatalogPage","commitId":"142edfde-7209-4d43-8faf-6867d3771290","commitTimeStamp":"2015-02-03T22:16:27.639269Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page329.json","@type":"CatalogPage","commitId":"62409c7e-5fd5-4cdc-a759-2db23d7ca23c","commitTimeStamp":"2015-02-03T22:25:09.7188179Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page330.json","@type":"CatalogPage","commitId":"29e08c85-8780-444e-ace5-91784d335399","commitTimeStamp":"2015-02-03T22:33:05.746775Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page331.json","@type":"CatalogPage","commitId":"06668c92-d893-4168-befc-511608b7ae3f","commitTimeStamp":"2015-02-03T22:42:11.1901533Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page332.json","@type":"CatalogPage","commitId":"772179bc-10c7-4dab-b831-ee2bfa8c5ffe","commitTimeStamp":"2015-02-03T22:52:30.4373933Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page333.json","@type":"CatalogPage","commitId":"f3cbe4b6-2172-4830-8a6d-70088dea0244","commitTimeStamp":"2015-02-03T23:01:12.7008656Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page334.json","@type":"CatalogPage","commitId":"686b407b-7cab-4079-ab5f-dd757d641ca4","commitTimeStamp":"2015-02-03T23:13:43.9819054Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page335.json","@type":"CatalogPage","commitId":"00d5c4b9-636b-45c6-a680-2d556d718a56","commitTimeStamp":"2015-02-03T23:25:05.1768076Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page336.json","@type":"CatalogPage","commitId":"306b99b7-0261-4e55-ab9a-178e566e7512","commitTimeStamp":"2015-02-03T23:42:26.3011262Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page337.json","@type":"CatalogPage","commitId":"aa16a7f2-516d-4bbe-971c-7ba13d2cba4b","commitTimeStamp":"2015-02-03T23:51:41.0000862Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page338.json","@type":"CatalogPage","commitId":"d1592bdb-7234-4814-a70b-e86f19632979","commitTimeStamp":"2015-02-04T00:00:40.5179104Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page339.json","@type":"CatalogPage","commitId":"03e83700-fd01-44b0-9983-3625e2dc818a","commitTimeStamp":"2015-02-04T00:08:25.5917181Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page340.json","@type":"CatalogPage","commitId":"e566c63a-0ff7-4e83-926d-dc5320d6af38","commitTimeStamp":"2015-02-04T00:16:30.5400644Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page341.json","@type":"CatalogPage","commitId":"d015c97e-4b5a-4ab9-9857-65cb63e5fdab","commitTimeStamp":"2015-02-04T00:23:53.7089945Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page342.json","@type":"CatalogPage","commitId":"4d857a9b-ac0e-49f9-bd58-d1667287c19f","commitTimeStamp":"2015-02-04T00:32:46.692667Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page343.json","@type":"CatalogPage","commitId":"a3872c69-9643-439a-962d-b29530bc6276","commitTimeStamp":"2015-02-04T00:44:50.5152894Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page344.json","@type":"CatalogPage","commitId":"fa9a8668-c431-42a0-ba42-53515225592b","commitTimeStamp":"2015-02-04T00:55:49.2311231Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page345.json","@type":"CatalogPage","commitId":"5c1c708c-7941-4ef5-97fd-a49c6cced30a","commitTimeStamp":"2015-02-04T01:03:04.2671669Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page346.json","@type":"CatalogPage","commitId":"4fc63ee1-400f-4eae-8b1d-c6089e2bbd83","commitTimeStamp":"2015-02-04T01:15:54.6441571Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page347.json","@type":"CatalogPage","commitId":"a05d069d-48e5-4724-a675-f8345f8571c8","commitTimeStamp":"2015-02-04T01:25:35.8958054Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page348.json","@type":"CatalogPage","commitId":"7edcacd8-7706-4ef5-99a5-a0c4063a297e","commitTimeStamp":"2015-02-04T01:35:19.9927612Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page349.json","@type":"CatalogPage","commitId":"90f92573-41c3-48a4-8ba2-70fb154f8ace","commitTimeStamp":"2015-02-04T01:44:26.6731494Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page350.json","@type":"CatalogPage","commitId":"3ab9c106-ab91-42a9-ab51-f6140d0b7abd","commitTimeStamp":"2015-02-04T01:54:52.5066144Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page351.json","@type":"CatalogPage","commitId":"d20174e5-d7eb-447a-9bec-9d942fcdfb9d","commitTimeStamp":"2015-02-04T02:05:39.2114664Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page352.json","@type":"CatalogPage","commitId":"d4c30b23-1522-46ac-a7c4-b15efd2fcdf1","commitTimeStamp":"2015-02-04T02:17:48.0302868Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page353.json","@type":"CatalogPage","commitId":"44554b20-c6ca-4bb3-ba28-fa65567c1960","commitTimeStamp":"2015-02-04T02:28:33.8234121Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page354.json","@type":"CatalogPage","commitId":"675362d2-d095-4b46-920c-a78214a6f230","commitTimeStamp":"2015-02-04T02:42:42.9793807Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page355.json","@type":"CatalogPage","commitId":"3db8e316-39b3-4b58-87ae-ce25be8929df","commitTimeStamp":"2015-02-04T02:52:27.5207087Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page356.json","@type":"CatalogPage","commitId":"8eb8eaec-5ec0-451d-bdf5-b8a9c841abd5","commitTimeStamp":"2015-02-04T03:02:53.7288686Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page357.json","@type":"CatalogPage","commitId":"5f22a1ac-599a-4b44-958d-6e3e708ca2d7","commitTimeStamp":"2015-02-04T03:15:47.4047914Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page358.json","@type":"CatalogPage","commitId":"37af8417-b9ac-4c77-afc1-31336501f7a5","commitTimeStamp":"2015-02-04T03:28:55.2524147Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page359.json","@type":"CatalogPage","commitId":"d1ed400d-004d-46c4-ae74-c295b89762ab","commitTimeStamp":"2015-02-04T03:44:11.6946144Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page360.json","@type":"CatalogPage","commitId":"798493c2-5b78-43d6-b07f-e1a3d838f4e6","commitTimeStamp":"2015-02-04T03:54:32.5270546Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page361.json","@type":"CatalogPage","commitId":"42f97080-1e53-4981-b7bb-00e9403dd738","commitTimeStamp":"2015-02-04T04:05:01.574091Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page362.json","@type":"CatalogPage","commitId":"6c2ee0e8-9d65-46e2-a81a-06b24bc01907","commitTimeStamp":"2015-02-04T04:15:19.8513013Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page363.json","@type":"CatalogPage","commitId":"8b881464-f678-442e-bd9e-069eaa5e49fd","commitTimeStamp":"2015-02-04T04:24:26.3088698Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page364.json","@type":"CatalogPage","commitId":"c4a3c3dc-1d26-4af8-8765-64acc905c2ff","commitTimeStamp":"2015-02-04T04:33:45.0426815Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page365.json","@type":"CatalogPage","commitId":"44c50531-df03-42ac-96a8-1b614412591d","commitTimeStamp":"2015-02-04T04:43:42.2434735Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page366.json","@type":"CatalogPage","commitId":"c033f651-1d8d-4f34-819b-9084353bb412","commitTimeStamp":"2015-02-04T04:54:34.5356612Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page367.json","@type":"CatalogPage","commitId":"8929e112-d0aa-43a7-8a9f-53245f9ee501","commitTimeStamp":"2015-02-04T05:21:59.6117509Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page368.json","@type":"CatalogPage","commitId":"4432cfa4-b83b-4e30-bdc3-751fe4345b4d","commitTimeStamp":"2015-02-04T05:33:12.376171Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page369.json","@type":"CatalogPage","commitId":"ee2055f6-2260-4b04-9861-944fb56c339d","commitTimeStamp":"2015-02-04T05:46:28.4357126Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page370.json","@type":"CatalogPage","commitId":"4deb155c-39e2-4e64-9c57-c25ea4e06e33","commitTimeStamp":"2015-02-04T05:58:03.5357251Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page371.json","@type":"CatalogPage","commitId":"9472efff-9915-466e-87c2-b27f94880c22","commitTimeStamp":"2015-02-04T06:22:09.9130954Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page372.json","@type":"CatalogPage","commitId":"bbf112f1-7680-488e-8295-b19bd7927a02","commitTimeStamp":"2015-02-04T06:38:43.1371942Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page373.json","@type":"CatalogPage","commitId":"4bdb32ef-a9fa-4629-bb4a-25c50c4861dc","commitTimeStamp":"2015-02-04T06:50:23.4379195Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page374.json","@type":"CatalogPage","commitId":"fdd0b72c-98ff-47fc-b05e-d2dea4ace82e","commitTimeStamp":"2015-02-04T07:02:03.7407579Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page375.json","@type":"CatalogPage","commitId":"13e23c20-7ae3-4a38-9395-5178d29f9b91","commitTimeStamp":"2015-02-04T07:19:48.746832Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page376.json","@type":"CatalogPage","commitId":"a7d78aae-2f1e-4a0a-8a77-fcf67e98e95e","commitTimeStamp":"2015-02-04T07:35:19.1372406Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page377.json","@type":"CatalogPage","commitId":"15c29c4a-f6a3-4ad7-b39b-a4f3cca456d9","commitTimeStamp":"2015-02-04T08:01:03.38493Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page378.json","@type":"CatalogPage","commitId":"6e1d7901-9643-41a5-b83b-ba6442d4d691","commitTimeStamp":"2015-02-04T08:11:21.9448264Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page379.json","@type":"CatalogPage","commitId":"5eda655c-059e-44ef-a64b-ac05fe73dace","commitTimeStamp":"2015-02-04T08:24:15.1657352Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page380.json","@type":"CatalogPage","commitId":"0de0dbc6-0282-491c-b9ce-b7cbcc8775d5","commitTimeStamp":"2015-02-04T08:36:45.0559321Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page381.json","@type":"CatalogPage","commitId":"1734333d-0b17-440e-b7d4-dcf6b6dd9cb0","commitTimeStamp":"2015-02-04T08:47:41.6684268Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page382.json","@type":"CatalogPage","commitId":"c54af2a5-2c03-424f-9018-4e1d439bb0d1","commitTimeStamp":"2015-02-04T09:00:14.370519Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page383.json","@type":"CatalogPage","commitId":"ccf4c334-e995-4b0e-bd19-be005ffb1ead","commitTimeStamp":"2015-02-04T09:11:50.6481556Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page384.json","@type":"CatalogPage","commitId":"7b8a9ef2-ac4b-49e2-b2b4-b88cdb3d48e0","commitTimeStamp":"2015-02-04T09:22:47.3948335Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page385.json","@type":"CatalogPage","commitId":"4df67eb1-7914-4892-8860-273edbd80c8c","commitTimeStamp":"2015-02-04T09:35:30.92205Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page386.json","@type":"CatalogPage","commitId":"ab9376c9-4603-4a93-89f6-788ebae06995","commitTimeStamp":"2015-02-04T09:48:26.0950683Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page387.json","@type":"CatalogPage","commitId":"201c7d42-6b0b-4706-a2cf-e83c7c413bf3","commitTimeStamp":"2015-02-04T10:03:03.9645052Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page388.json","@type":"CatalogPage","commitId":"01aa7bb5-f80d-42dc-9df5-0670004d917c","commitTimeStamp":"2015-02-04T10:15:55.335089Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page389.json","@type":"CatalogPage","commitId":"58da8f4a-ed59-484e-b5e6-b687254541bc","commitTimeStamp":"2015-02-04T10:32:14.3350885Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page390.json","@type":"CatalogPage","commitId":"8abfaead-37d0-445e-b434-7454f54d0d95","commitTimeStamp":"2015-02-04T10:46:50.37556Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page391.json","@type":"CatalogPage","commitId":"d6aca6ce-1227-45f3-9465-5e615197c540","commitTimeStamp":"2015-02-04T11:02:25.9173299Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page392.json","@type":"CatalogPage","commitId":"70f57ade-229a-44fe-af6c-642d60c0c96e","commitTimeStamp":"2015-02-04T11:36:56.4947868Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page393.json","@type":"CatalogPage","commitId":"6a900836-23ba-495c-969c-fbadabea7c6d","commitTimeStamp":"2015-02-04T11:49:16.8251866Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page394.json","@type":"CatalogPage","commitId":"748cec8f-2600-4663-a2d3-9b701a8cb880","commitTimeStamp":"2015-02-04T12:04:54.7702584Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page395.json","@type":"CatalogPage","commitId":"8a5c60cb-a439-4f1c-9a1b-ff67de265cac","commitTimeStamp":"2015-02-04T12:17:08.4525071Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page396.json","@type":"CatalogPage","commitId":"0264aabd-189d-4e3c-a381-841d72a2b4df","commitTimeStamp":"2015-02-04T12:30:01.706696Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page397.json","@type":"CatalogPage","commitId":"9e015379-9b6b-4926-9422-c78ddf891b3d","commitTimeStamp":"2015-02-04T12:41:49.2439609Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page398.json","@type":"CatalogPage","commitId":"86a4653a-8fa4-43cf-83bc-f4ecdef9fb0b","commitTimeStamp":"2015-02-04T12:54:17.7933123Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page399.json","@type":"CatalogPage","commitId":"0ae738b4-9983-41e6-a399-4cfe37875461","commitTimeStamp":"2015-02-04T13:04:31.4611357Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page400.json","@type":"CatalogPage","commitId":"1c5ad5d7-d676-4b21-bb54-d23203bf5f5e","commitTimeStamp":"2015-02-04T13:16:58.2474328Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page401.json","@type":"CatalogPage","commitId":"f4aea808-59e6-42eb-9960-c769b5c5ff41","commitTimeStamp":"2015-02-04T13:30:26.7133985Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page402.json","@type":"CatalogPage","commitId":"7d2f07f4-0e5e-4308-ad99-cadcdb2e45a3","commitTimeStamp":"2015-02-04T13:43:31.9414407Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page403.json","@type":"CatalogPage","commitId":"778b4c6c-59ee-460b-9cd9-abde85c255a0","commitTimeStamp":"2015-02-04T13:55:50.6642285Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page404.json","@type":"CatalogPage","commitId":"fbc2bf18-c2b6-4c11-963f-bc1fc29b8288","commitTimeStamp":"2015-02-04T14:06:42.6924634Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page405.json","@type":"CatalogPage","commitId":"c46b1bd8-e930-4ac9-a74c-6ea1850cbf46","commitTimeStamp":"2015-02-04T14:17:44.5448215Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page406.json","@type":"CatalogPage","commitId":"9b0bfdfc-5716-4c5e-8fa7-989be121e983","commitTimeStamp":"2015-02-04T14:31:12.4117817Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page407.json","@type":"CatalogPage","commitId":"d21efa49-0e25-4f39-bf46-7f63031934d8","commitTimeStamp":"2015-02-04T14:41:39.762887Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page408.json","@type":"CatalogPage","commitId":"c30b3a4a-a7a4-4c56-aed5-6999db801322","commitTimeStamp":"2015-02-04T14:53:54.3764241Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page409.json","@type":"CatalogPage","commitId":"f7ebfbc8-b7ee-4eda-9870-580b01c69cd3","commitTimeStamp":"2015-02-04T15:06:16.8034283Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page410.json","@type":"CatalogPage","commitId":"27f6847f-ee84-40f3-a77c-c5992bfc0c10","commitTimeStamp":"2015-02-04T15:20:38.0648912Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page411.json","@type":"CatalogPage","commitId":"714106eb-0d3c-44b9-9b0a-29e8099f1668","commitTimeStamp":"2015-02-04T15:33:29.8502042Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page412.json","@type":"CatalogPage","commitId":"d89de50f-63d0-4c8a-9feb-fa4b1d745d2a","commitTimeStamp":"2015-02-04T15:47:47.6944399Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page413.json","@type":"CatalogPage","commitId":"0ac87413-689b-43a4-80f6-4ee0a2eeba40","commitTimeStamp":"2015-02-04T16:04:30.0517037Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page414.json","@type":"CatalogPage","commitId":"4b95bf05-a4ce-42e4-b4ec-86332376fafc","commitTimeStamp":"2015-02-04T16:19:23.7699735Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page415.json","@type":"CatalogPage","commitId":"9a1de5ac-dd83-4c49-abd3-a4b4088027bb","commitTimeStamp":"2015-02-04T16:32:32.6969168Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page416.json","@type":"CatalogPage","commitId":"e88ccc08-b451-4ce0-a344-03a1a0a1074b","commitTimeStamp":"2015-02-04T16:46:17.3703575Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page417.json","@type":"CatalogPage","commitId":"06cf846c-9e82-4fd8-92bd-6b2a0fb2b92b","commitTimeStamp":"2015-02-04T16:59:03.7318501Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page418.json","@type":"CatalogPage","commitId":"4d6c65cd-55b4-42c7-82cf-c819ffcb9005","commitTimeStamp":"2015-02-04T17:12:35.2640535Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page419.json","@type":"CatalogPage","commitId":"239a714d-f1e4-430d-9bf5-a4a7571aa40a","commitTimeStamp":"2015-02-04T17:24:50.1245886Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page420.json","@type":"CatalogPage","commitId":"d998ad45-31d6-46f6-8ba1-8f91b2d10387","commitTimeStamp":"2015-02-04T17:37:14.6694433Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page421.json","@type":"CatalogPage","commitId":"7d0a9a54-fe44-4756-b3c7-9b96c5b20aef","commitTimeStamp":"2015-02-04T17:47:38.9791836Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page422.json","@type":"CatalogPage","commitId":"1d9481a6-73db-48e1-b224-34df475627ed","commitTimeStamp":"2015-02-04T17:59:10.9599962Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page423.json","@type":"CatalogPage","commitId":"1e446b03-3e5d-4a90-9edb-7aafe1fee419","commitTimeStamp":"2015-02-04T18:10:26.0045231Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page424.json","@type":"CatalogPage","commitId":"d0cac5c7-a885-4df4-82c0-5368db8d6dbe","commitTimeStamp":"2015-02-04T18:21:37.9929941Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page425.json","@type":"CatalogPage","commitId":"6b7f85f4-3a5e-492a-a566-9154c94e09a8","commitTimeStamp":"2015-02-04T18:35:04.8514372Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page426.json","@type":"CatalogPage","commitId":"c9fc28a1-7a18-4a9b-928b-4b9cca10e037","commitTimeStamp":"2015-02-04T18:47:25.9448342Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page427.json","@type":"CatalogPage","commitId":"9849abd0-84b6-44c5-9d02-58b3e7ae3550","commitTimeStamp":"2015-02-04T18:59:40.2683602Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page428.json","@type":"CatalogPage","commitId":"7ea4b90b-0739-4f34-a9d1-40ccae9b5f22","commitTimeStamp":"2015-02-04T19:11:24.3505991Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page429.json","@type":"CatalogPage","commitId":"0659dcc0-b637-4063-a799-ef1d4d182df9","commitTimeStamp":"2015-02-04T19:26:13.3516349Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page430.json","@type":"CatalogPage","commitId":"e67cbec9-7416-4d5a-bc67-84a3b5ab84e7","commitTimeStamp":"2015-02-04T19:36:00.6926103Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page431.json","@type":"CatalogPage","commitId":"26ba593c-54f7-4b1d-9518-a16cc5a122a5","commitTimeStamp":"2015-02-04T19:48:29.6882102Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page432.json","@type":"CatalogPage","commitId":"be7c72f2-fa1e-4106-9808-09de8c00e6fa","commitTimeStamp":"2015-02-04T19:59:36.7763098Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page433.json","@type":"CatalogPage","commitId":"311bacaa-d027-4357-a4e3-b4f96596f4d3","commitTimeStamp":"2015-02-04T20:10:32.3821632Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page434.json","@type":"CatalogPage","commitId":"b54fa66d-c1a1-49b6-9efc-904575c70695","commitTimeStamp":"2015-02-04T20:31:05.0513587Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page435.json","@type":"CatalogPage","commitId":"7f2a6bc1-9d6a-46bb-aeff-3d8246cd1026","commitTimeStamp":"2015-02-04T20:44:28.4212251Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page436.json","@type":"CatalogPage","commitId":"befd724f-543c-478b-b4cb-180604a37c42","commitTimeStamp":"2015-02-04T20:58:27.1081937Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page437.json","@type":"CatalogPage","commitId":"d4180d17-cd82-4e03-a330-88da89597d5a","commitTimeStamp":"2015-02-04T21:11:42.1672067Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page438.json","@type":"CatalogPage","commitId":"f70d672a-3c27-40e3-a4db-980abbf53cfd","commitTimeStamp":"2015-02-04T21:22:48.9210026Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page439.json","@type":"CatalogPage","commitId":"af279cfa-9395-46f3-9501-bc93db070246","commitTimeStamp":"2015-02-04T21:34:10.0274114Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page440.json","@type":"CatalogPage","commitId":"790c2eff-c7f4-4f51-97d6-8486002f2452","commitTimeStamp":"2015-02-04T21:43:37.1271435Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page441.json","@type":"CatalogPage","commitId":"5944d326-7865-4d9b-846b-14a6c7913506","commitTimeStamp":"2015-02-04T21:55:33.0499548Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page442.json","@type":"CatalogPage","commitId":"4e8a0d3c-04d6-4264-878a-5e958af32cb1","commitTimeStamp":"2015-02-04T22:07:30.4060687Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page443.json","@type":"CatalogPage","commitId":"65953040-01c1-4d5b-abc7-22db573452b3","commitTimeStamp":"2015-02-04T22:21:58.4880127Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page444.json","@type":"CatalogPage","commitId":"d55dff0d-4fc0-4114-906e-3d5e266621d7","commitTimeStamp":"2015-02-04T22:34:06.5387385Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page445.json","@type":"CatalogPage","commitId":"2d538415-a7c3-4394-a3ae-f687b4bc577d","commitTimeStamp":"2015-02-04T22:46:40.1271169Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page446.json","@type":"CatalogPage","commitId":"dce24475-b506-4460-8994-e89c3e0c443d","commitTimeStamp":"2015-02-04T22:57:53.5705273Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page447.json","@type":"CatalogPage","commitId":"f946be3a-9837-42e0-a3f9-b2b0590c9df3","commitTimeStamp":"2015-02-04T23:09:07.2438388Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page448.json","@type":"CatalogPage","commitId":"2a28b126-1641-49d9-8c28-72573932569d","commitTimeStamp":"2015-02-04T23:22:08.6450592Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page449.json","@type":"CatalogPage","commitId":"d118d21d-acf2-401a-ab4f-8663ea7f4b14","commitTimeStamp":"2015-02-04T23:35:04.5425541Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page450.json","@type":"CatalogPage","commitId":"d0c87d73-17b4-4745-aa81-acc25d0d0f71","commitTimeStamp":"2015-02-04T23:47:48.0061055Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page451.json","@type":"CatalogPage","commitId":"be0d1a43-15e4-4dc1-bca0-e0cd6b021aa9","commitTimeStamp":"2015-02-05T00:05:12.9556301Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page452.json","@type":"CatalogPage","commitId":"63a8c243-e41f-48ca-93d9-058588694fdd","commitTimeStamp":"2015-02-05T00:21:32.2493868Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page453.json","@type":"CatalogPage","commitId":"b4461b85-87f8-41ab-b754-22401caca71c","commitTimeStamp":"2015-02-05T00:36:48.8196356Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page454.json","@type":"CatalogPage","commitId":"1aefad59-a000-45be-a21a-8da15db9eea8","commitTimeStamp":"2015-02-05T00:50:26.4579058Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page455.json","@type":"CatalogPage","commitId":"af58a20a-f4a2-4802-9cd0-630925a59f05","commitTimeStamp":"2015-02-05T01:03:40.5594651Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page456.json","@type":"CatalogPage","commitId":"807ed9d9-7ec8-4f8c-9b3f-d5797eabfdb0","commitTimeStamp":"2015-02-05T01:17:12.6648449Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page457.json","@type":"CatalogPage","commitId":"5ff2fb89-d056-45b6-93a5-6db852f113a7","commitTimeStamp":"2015-02-05T01:28:03.2901165Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page458.json","@type":"CatalogPage","commitId":"91805cf2-1bd1-49c2-8ec0-b13ace91a569","commitTimeStamp":"2015-02-05T01:43:01.2569697Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page459.json","@type":"CatalogPage","commitId":"c0ecdcd6-d5ae-4443-892a-bb5d83e38789","commitTimeStamp":"2015-02-05T01:54:58.8272167Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page460.json","@type":"CatalogPage","commitId":"d0f7fb07-1780-4e20-a2b5-ac59caa93536","commitTimeStamp":"2015-02-05T02:05:24.1892056Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page461.json","@type":"CatalogPage","commitId":"9ec0f531-8051-4a12-82e5-a3f8ce09e1c3","commitTimeStamp":"2015-02-05T02:17:33.5499733Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page462.json","@type":"CatalogPage","commitId":"c1ddf138-e93f-4925-ba01-b78539f8f778","commitTimeStamp":"2015-02-05T02:33:38.5622561Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page463.json","@type":"CatalogPage","commitId":"bd2edbf9-0576-41f8-b7f8-01c891d42d3c","commitTimeStamp":"2015-02-05T02:47:52.4787772Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page464.json","@type":"CatalogPage","commitId":"a0458429-6994-4594-8d1d-d0aa3aeb235f","commitTimeStamp":"2015-02-05T03:03:45.6117836Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page465.json","@type":"CatalogPage","commitId":"fc1b5823-e541-4e78-9783-51849705ddba","commitTimeStamp":"2015-02-05T03:23:54.5620828Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page466.json","@type":"CatalogPage","commitId":"ff89bb42-f0f4-4e68-8b4e-fa8bb6888874","commitTimeStamp":"2015-02-05T03:39:12.4451214Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page467.json","@type":"CatalogPage","commitId":"2d37ee36-7a8b-4bd8-8cbd-b80c7327f943","commitTimeStamp":"2015-02-05T03:53:04.1990737Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page468.json","@type":"CatalogPage","commitId":"b155e151-5a31-46dd-9107-d23576c4f219","commitTimeStamp":"2015-02-05T04:07:54.6020069Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page469.json","@type":"CatalogPage","commitId":"b4c88bb8-aea2-49de-97ba-8ab1dcdecb11","commitTimeStamp":"2015-02-05T04:20:30.0443412Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page470.json","@type":"CatalogPage","commitId":"a3bc1f2f-61cf-408a-aa45-7a5c6fc49fb0","commitTimeStamp":"2015-02-05T04:32:06.3250687Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page471.json","@type":"CatalogPage","commitId":"946bdfd7-a7a2-4b18-a8ce-1ec494f19ebf","commitTimeStamp":"2015-02-05T04:42:40.9929423Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page472.json","@type":"CatalogPage","commitId":"d6f13103-dea5-439c-938b-87a0a11e0176","commitTimeStamp":"2015-02-05T04:54:48.6401829Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page473.json","@type":"CatalogPage","commitId":"51e4f890-abbc-403e-a1c2-c1d784f721db","commitTimeStamp":"2015-02-05T05:07:02.4099991Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page474.json","@type":"CatalogPage","commitId":"3dce58d3-cb3c-46c2-ba7f-4bc6bad56cbf","commitTimeStamp":"2015-02-05T05:20:34.7724144Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page475.json","@type":"CatalogPage","commitId":"fcddc850-2077-4bd6-a8de-f8269b3ca3ce","commitTimeStamp":"2015-02-05T05:31:11.6480563Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page476.json","@type":"CatalogPage","commitId":"5d09f505-55e8-40b5-baa0-b8baa3fbe149","commitTimeStamp":"2015-02-05T05:41:24.1546918Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page477.json","@type":"CatalogPage","commitId":"9cbc034d-4e58-4e60-9615-4610e538ed87","commitTimeStamp":"2015-02-05T05:53:25.4642977Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page478.json","@type":"CatalogPage","commitId":"bfd8412d-9bc0-4146-a68e-31743db8e57a","commitTimeStamp":"2015-02-05T06:04:18.1157119Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page479.json","@type":"CatalogPage","commitId":"89c81e30-d7a8-42d3-b11b-5b607446173e","commitTimeStamp":"2015-02-05T06:15:29.2498054Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page480.json","@type":"CatalogPage","commitId":"0deb6fad-14d7-44f7-b479-527b35495461","commitTimeStamp":"2015-02-05T06:28:20.0956887Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page481.json","@type":"CatalogPage","commitId":"c822e813-fb0b-4f31-8500-e70082f4b055","commitTimeStamp":"2015-02-05T06:37:58.9025458Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page482.json","@type":"CatalogPage","commitId":"1735f92c-b915-4b2e-be69-221fc2bb9fc7","commitTimeStamp":"2015-02-05T06:49:23.2463711Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page483.json","@type":"CatalogPage","commitId":"ef2f54e5-ea17-49ce-a6a2-15e54cc8e24a","commitTimeStamp":"2015-02-05T07:01:44.3257251Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page484.json","@type":"CatalogPage","commitId":"4f071553-33d1-40a0-981d-15c8a277ff7b","commitTimeStamp":"2015-02-05T07:13:03.3234195Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page485.json","@type":"CatalogPage","commitId":"95aa409a-1f13-4400-a8b0-76a1daf127c3","commitTimeStamp":"2015-02-05T07:23:56.8796314Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page486.json","@type":"CatalogPage","commitId":"448d7583-668c-49b5-82ba-b1d8475cb55f","commitTimeStamp":"2015-02-05T07:37:56.423408Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page487.json","@type":"CatalogPage","commitId":"646e914d-ec9a-46ba-af30-b6b27d5275bd","commitTimeStamp":"2015-02-05T07:49:26.5403309Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page488.json","@type":"CatalogPage","commitId":"093cfeae-7e71-44d9-8c1a-ca32c10cdbdb","commitTimeStamp":"2015-02-05T08:01:01.3653331Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page489.json","@type":"CatalogPage","commitId":"21f3c7f2-27dc-423a-94d9-a6408958337d","commitTimeStamp":"2015-02-05T08:17:15.782181Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page490.json","@type":"CatalogPage","commitId":"14ffc57b-b1f9-48d1-b705-487e9e15d8d6","commitTimeStamp":"2015-02-05T08:28:26.0343041Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page491.json","@type":"CatalogPage","commitId":"a676990e-a68f-4c2e-aab5-16eb8db651b9","commitTimeStamp":"2015-02-05T08:39:47.8196395Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page492.json","@type":"CatalogPage","commitId":"c9dc2f01-bd06-4541-b193-0d95ac9d3686","commitTimeStamp":"2015-02-05T08:51:24.199278Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page493.json","@type":"CatalogPage","commitId":"1fe61f96-7100-4899-a7e8-d12b6ba5a5d9","commitTimeStamp":"2015-02-05T09:00:00.8928332Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page494.json","@type":"CatalogPage","commitId":"cd75e2cd-681f-42cb-a3cb-cbef0c13da3d","commitTimeStamp":"2015-02-05T09:09:28.1955232Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page495.json","@type":"CatalogPage","commitId":"ed208279-7483-465f-9cb7-06366823d4e5","commitTimeStamp":"2015-02-05T09:21:02.046305Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page496.json","@type":"CatalogPage","commitId":"647b829c-63ae-4c16-b4f1-91c82088d58b","commitTimeStamp":"2015-02-05T09:34:16.7750559Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page497.json","@type":"CatalogPage","commitId":"46a361d2-2b06-46b9-9c47-b527ad6af1d2","commitTimeStamp":"2015-02-05T09:45:57.183807Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page498.json","@type":"CatalogPage","commitId":"8affb539-363e-4050-a007-a03a53e348a6","commitTimeStamp":"2015-02-05T09:59:44.1172593Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page499.json","@type":"CatalogPage","commitId":"19ce2f3f-242c-4164-b42e-35db3b746be1","commitTimeStamp":"2015-02-05T10:08:39.4010251Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page500.json","@type":"CatalogPage","commitId":"d1c9dfe7-3189-40c9-baee-04e1e9d5cdc7","commitTimeStamp":"2015-02-05T10:17:57.4181877Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page501.json","@type":"CatalogPage","commitId":"581dde51-3d21-400e-98c1-28ef3397154e","commitTimeStamp":"2015-02-05T10:29:20.1042386Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page502.json","@type":"CatalogPage","commitId":"26f66a25-7725-4a74-b5dc-b2b897c40f10","commitTimeStamp":"2015-02-05T10:40:55.6065182Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page503.json","@type":"CatalogPage","commitId":"bf3f32db-9ccc-4d20-be10-269a3230475a","commitTimeStamp":"2015-02-05T10:51:58.894143Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page504.json","@type":"CatalogPage","commitId":"df859c67-1e2e-48a1-b833-a62df30393ab","commitTimeStamp":"2015-02-05T11:04:27.3280347Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page505.json","@type":"CatalogPage","commitId":"66df1fb6-e939-4e7a-a195-a0673e2110fe","commitTimeStamp":"2015-02-05T11:17:08.7478522Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page506.json","@type":"CatalogPage","commitId":"102105f4-12c4-4fb6-94cc-4ec2ff318efc","commitTimeStamp":"2015-02-05T11:30:20.4962932Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page507.json","@type":"CatalogPage","commitId":"2cbb0399-984c-4b43-90aa-456cafb10cfb","commitTimeStamp":"2015-02-05T11:43:51.7345454Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page508.json","@type":"CatalogPage","commitId":"a493ad1e-560b-48c6-88ad-33c731bf0996","commitTimeStamp":"2015-02-05T11:54:37.8400733Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page509.json","@type":"CatalogPage","commitId":"ce68c992-bc29-44af-959b-ea70ea6307ad","commitTimeStamp":"2015-02-05T12:07:39.9265539Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page510.json","@type":"CatalogPage","commitId":"1e15f0a5-1fa0-4bd0-a07d-0f18ef2ffc09","commitTimeStamp":"2015-02-05T12:20:38.6038816Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page511.json","@type":"CatalogPage","commitId":"806508f3-18e3-47dc-acd2-55ad79ac596e","commitTimeStamp":"2015-02-05T12:31:24.5200813Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page512.json","@type":"CatalogPage","commitId":"40c23673-e359-4f63-b91c-2d4d45d8c488","commitTimeStamp":"2015-02-05T12:44:34.6392587Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page513.json","@type":"CatalogPage","commitId":"11d32072-50f4-4e47-abce-176b0dc27cca","commitTimeStamp":"2015-02-05T12:59:33.3353062Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page514.json","@type":"CatalogPage","commitId":"238106a5-c02e-43a1-9fed-3133ef9b62d7","commitTimeStamp":"2015-02-05T13:12:09.4297375Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page515.json","@type":"CatalogPage","commitId":"0bd10a9f-5449-42d7-87f3-073296e36ed0","commitTimeStamp":"2015-02-05T13:22:09.8886745Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page516.json","@type":"CatalogPage","commitId":"dc2674b0-7188-4c64-b389-ccb7a5ba5f7a","commitTimeStamp":"2015-02-05T13:35:40.9389477Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page517.json","@type":"CatalogPage","commitId":"86ef842a-0fe1-4506-a873-2571484b0567","commitTimeStamp":"2015-02-05T13:47:56.8199506Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page518.json","@type":"CatalogPage","commitId":"5b8f8c0e-878e-45ed-995a-861490898f6f","commitTimeStamp":"2015-02-05T14:01:39.05619Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page519.json","@type":"CatalogPage","commitId":"d176e111-5cd3-4047-bc5b-cb82eab33be0","commitTimeStamp":"2015-02-05T14:14:22.196008Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page520.json","@type":"CatalogPage","commitId":"8e6413ee-7cca-4be7-be77-efdc78e1c415","commitTimeStamp":"2015-02-05T14:26:12.9548316Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page521.json","@type":"CatalogPage","commitId":"88dd9b2f-9491-465a-ba55-5ce3224e067d","commitTimeStamp":"2015-02-05T14:40:27.9953061Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page522.json","@type":"CatalogPage","commitId":"985cae97-3e2a-4963-8103-1055949de92e","commitTimeStamp":"2015-02-05T14:52:08.0613686Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page523.json","@type":"CatalogPage","commitId":"67c3d233-fd5d-4b22-8af0-5a0213d1d697","commitTimeStamp":"2015-02-05T15:04:43.3233548Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page524.json","@type":"CatalogPage","commitId":"63611df0-a3b3-4764-b45c-42c52d435b2e","commitTimeStamp":"2015-02-05T15:15:03.8469073Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page525.json","@type":"CatalogPage","commitId":"d8172229-80dc-4f2f-b58e-c93f06bdbe31","commitTimeStamp":"2015-02-05T15:25:18.1165808Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page526.json","@type":"CatalogPage","commitId":"0af8d7b5-6fca-4295-957d-bda1624b1bdf","commitTimeStamp":"2015-02-05T15:35:14.1144786Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page527.json","@type":"CatalogPage","commitId":"75487f2b-a4f9-4d73-9287-e886fa62b3e6","commitTimeStamp":"2015-02-05T15:46:43.8537843Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page528.json","@type":"CatalogPage","commitId":"7bd88a22-9ff9-4a71-864d-9de51a9df513","commitTimeStamp":"2015-02-05T15:56:16.6922894Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page529.json","@type":"CatalogPage","commitId":"e8a4d86d-9797-48de-a1d2-dda381327418","commitTimeStamp":"2015-02-05T16:05:14.1055993Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page530.json","@type":"CatalogPage","commitId":"ecd7df28-6da1-4098-8594-7cc94682a5bb","commitTimeStamp":"2015-02-05T16:17:24.2037667Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page531.json","@type":"CatalogPage","commitId":"f599a49f-d3f9-4c45-8753-b9bc20b8cf3b","commitTimeStamp":"2015-02-05T16:30:33.7417353Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page532.json","@type":"CatalogPage","commitId":"28915472-3057-46d4-b3fd-0d19ee1fae7c","commitTimeStamp":"2015-02-05T16:42:11.1811432Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page533.json","@type":"CatalogPage","commitId":"d703cf18-8fd3-41d5-af0d-2d4452be466a","commitTimeStamp":"2015-02-05T16:53:57.9461452Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page534.json","@type":"CatalogPage","commitId":"6adbb18a-c83c-4299-860c-b704e813d1fa","commitTimeStamp":"2015-02-05T17:06:50.1226679Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page535.json","@type":"CatalogPage","commitId":"141b13a5-cb9e-4730-94a3-08a689783f85","commitTimeStamp":"2015-02-05T17:21:33.9793458Z","count":537},{"@id":"https://api.nuget.org/v3/catalog0/page536.json","@type":"CatalogPage","commitId":"97e5932a-4973-4fd0-98ff-3cbdb1ac4d0e","commitTimeStamp":"2015-02-05T17:35:24.8169132Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page537.json","@type":"CatalogPage","commitId":"bfbd711c-e2d5-4fe1-b1dd-7f4bec928cbc","commitTimeStamp":"2015-02-05T17:48:04.0383919Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page538.json","@type":"CatalogPage","commitId":"f55a7dc7-ab99-4bb4-9f72-74fc20bd33d1","commitTimeStamp":"2015-02-05T18:00:23.4360191Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page539.json","@type":"CatalogPage","commitId":"6b857720-6f42-41c5-aa76-0e9dc0bbb23f","commitTimeStamp":"2015-02-05T18:16:00.6210045Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page540.json","@type":"CatalogPage","commitId":"0e0eb9a0-0467-42ee-a5dc-45e50d175887","commitTimeStamp":"2015-02-05T18:29:27.3280424Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page541.json","@type":"CatalogPage","commitId":"5a2912a3-cc09-4a36-9cdd-46841e79e33e","commitTimeStamp":"2015-02-05T18:43:02.930595Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page542.json","@type":"CatalogPage","commitId":"261db014-c4ad-44f5-a5ab-d4ca40b8a875","commitTimeStamp":"2015-02-05T18:54:09.7076788Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page543.json","@type":"CatalogPage","commitId":"5636c8b9-72d3-4a9d-9d0b-4cf629851654","commitTimeStamp":"2015-02-05T19:05:56.9800317Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page544.json","@type":"CatalogPage","commitId":"973fe84d-8d15-4723-8f03-1a949ac42832","commitTimeStamp":"2015-02-05T19:16:58.1685472Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page545.json","@type":"CatalogPage","commitId":"e9c01248-426d-4949-a961-805d3dc8689b","commitTimeStamp":"2015-02-05T19:30:43.1943508Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page546.json","@type":"CatalogPage","commitId":"3976c500-f0e2-4b85-97fc-1d691bae407d","commitTimeStamp":"2015-02-05T19:43:48.5545712Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page547.json","@type":"CatalogPage","commitId":"a7018bb3-d621-4d83-bfb0-1f48e88a9a7b","commitTimeStamp":"2015-02-05T19:56:29.9518347Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page548.json","@type":"CatalogPage","commitId":"8726b157-e8a2-4ca6-a592-6b01da0344c0","commitTimeStamp":"2015-02-05T20:07:38.4967986Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page549.json","@type":"CatalogPage","commitId":"a6279aa7-7d16-4e6b-9a11-888f68d9f9fb","commitTimeStamp":"2015-02-05T20:19:41.7601267Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page550.json","@type":"CatalogPage","commitId":"8ef674cf-2f37-462f-a909-b44a5db90cc2","commitTimeStamp":"2015-02-05T20:31:45.802759Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page551.json","@type":"CatalogPage","commitId":"f3dfd823-5ee8-4c83-be97-0fcbdea4d98e","commitTimeStamp":"2015-02-05T20:43:32.8823248Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page552.json","@type":"CatalogPage","commitId":"87c1bced-6e95-4e67-afb0-b6c70d796ce8","commitTimeStamp":"2015-02-05T20:56:45.8856377Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page553.json","@type":"CatalogPage","commitId":"6afad203-8e89-4e82-9227-c48df9db3e28","commitTimeStamp":"2015-02-05T21:09:09.2884731Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page554.json","@type":"CatalogPage","commitId":"e7af090f-79c6-4755-928d-31c7c3765f54","commitTimeStamp":"2015-02-05T21:23:41.3726676Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page555.json","@type":"CatalogPage","commitId":"6674b872-075b-49d8-b132-474675f7708a","commitTimeStamp":"2015-02-05T21:36:26.5396136Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page556.json","@type":"CatalogPage","commitId":"d719fc5f-ac47-4584-80de-8b745f86bbe6","commitTimeStamp":"2015-02-05T21:48:15.2739527Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page557.json","@type":"CatalogPage","commitId":"34dbc756-46e4-4b0b-b986-2b4807a59539","commitTimeStamp":"2015-02-05T22:01:05.9239933Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page558.json","@type":"CatalogPage","commitId":"20ee33c0-eab1-4a1d-8e26-30e3ca022f0c","commitTimeStamp":"2015-02-05T22:17:51.0305352Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page559.json","@type":"CatalogPage","commitId":"49a5f6aa-3d50-460d-a359-6d959dde9b13","commitTimeStamp":"2015-02-05T22:38:45.3953305Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page560.json","@type":"CatalogPage","commitId":"78d719f7-de81-4457-83b8-3fc7de2f95af","commitTimeStamp":"2015-02-05T22:58:19.7399384Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page561.json","@type":"CatalogPage","commitId":"514c19d5-d726-4554-9273-75c51bfe0931","commitTimeStamp":"2015-02-05T23:10:23.8542721Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page562.json","@type":"CatalogPage","commitId":"cf0ca8a2-bbcf-4038-b77c-8d0198f56cfc","commitTimeStamp":"2015-02-05T23:22:39.2222835Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page563.json","@type":"CatalogPage","commitId":"f8544c16-5fd5-4f5b-abaf-8a1228f9cad1","commitTimeStamp":"2015-02-05T23:38:00.4686543Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page564.json","@type":"CatalogPage","commitId":"4b10372f-6b85-4cf0-9d42-8cb1b24aa61c","commitTimeStamp":"2015-02-05T23:49:18.8881366Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page565.json","@type":"CatalogPage","commitId":"d4d07855-bcf2-492a-a451-3597375a1f49","commitTimeStamp":"2015-02-06T00:01:24.9467785Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page566.json","@type":"CatalogPage","commitId":"b5dfbeb2-9d9b-4bea-9971-36b3e5693e68","commitTimeStamp":"2015-02-06T00:28:28.695396Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page567.json","@type":"CatalogPage","commitId":"c7b761b2-4829-4e62-a2ad-2ab36022eee1","commitTimeStamp":"2015-02-06T00:44:18.3444813Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page568.json","@type":"CatalogPage","commitId":"ef8f2597-16a4-4136-88f5-cfa9bffe49c4","commitTimeStamp":"2015-02-06T00:58:02.7884167Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page569.json","@type":"CatalogPage","commitId":"335710f3-4c69-4c53-99bc-b52f679ff4bb","commitTimeStamp":"2015-02-06T01:09:34.2267782Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page570.json","@type":"CatalogPage","commitId":"369bd99d-d3ee-421f-9d11-39b951ca18f2","commitTimeStamp":"2015-02-06T01:40:47.3612342Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page571.json","@type":"CatalogPage","commitId":"34ee0d73-6e41-43b8-b9e6-a1be25d3869b","commitTimeStamp":"2015-02-06T02:08:18.183294Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page572.json","@type":"CatalogPage","commitId":"c28a9c7a-a657-442c-8b33-2bc099533a1c","commitTimeStamp":"2015-02-06T02:22:45.7203859Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page573.json","@type":"CatalogPage","commitId":"bf1cbaaf-8555-4be7-ab72-34f320fbb243","commitTimeStamp":"2015-02-06T02:52:04.5560323Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page574.json","@type":"CatalogPage","commitId":"1904d7c8-c883-4e49-aa45-f22aee3137da","commitTimeStamp":"2015-02-06T03:04:09.7302948Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page575.json","@type":"CatalogPage","commitId":"ccbb8618-5bc5-4ca2-94aa-e0056f38bcac","commitTimeStamp":"2015-02-06T03:17:28.5649977Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page576.json","@type":"CatalogPage","commitId":"f8799308-92c0-488b-8178-2ec114bbb9c1","commitTimeStamp":"2015-02-06T03:30:39.620365Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page577.json","@type":"CatalogPage","commitId":"52381148-507a-43d2-a9a4-611859984b3d","commitTimeStamp":"2015-02-06T03:44:35.5345929Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page578.json","@type":"CatalogPage","commitId":"cfccf61e-04ee-4198-b78d-09ffb168c8de","commitTimeStamp":"2015-02-06T04:01:56.432777Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page579.json","@type":"CatalogPage","commitId":"e1c6183b-4684-4b90-a272-3e656f96abd5","commitTimeStamp":"2015-02-06T04:17:10.4130995Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page580.json","@type":"CatalogPage","commitId":"e656f989-ca47-40ab-8b7f-160d187f8ec3","commitTimeStamp":"2015-02-06T04:32:37.7962027Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page581.json","@type":"CatalogPage","commitId":"89947ce8-e286-4d61-ad2c-9b9ddb13e9b4","commitTimeStamp":"2015-02-06T04:44:57.7715254Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page582.json","@type":"CatalogPage","commitId":"b634f919-a34d-4306-ac4b-35f21bfea6a8","commitTimeStamp":"2015-02-06T04:57:18.3251858Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page583.json","@type":"CatalogPage","commitId":"5ddf6537-ed13-4974-91a2-edf8b5a1b940","commitTimeStamp":"2015-02-06T05:13:14.1632913Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page584.json","@type":"CatalogPage","commitId":"35e38d66-192a-40ae-88d8-16427d45d010","commitTimeStamp":"2015-02-06T05:24:08.52512Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page585.json","@type":"CatalogPage","commitId":"8e0c0839-553e-425c-a189-11600e8cc5c1","commitTimeStamp":"2015-02-06T05:39:28.7766115Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page586.json","@type":"CatalogPage","commitId":"b81a02d0-aaa8-4a4b-8715-ad9aad8f2808","commitTimeStamp":"2015-02-06T05:50:48.8664383Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page587.json","@type":"CatalogPage","commitId":"0eb7a752-afe2-4ac6-abf2-8ec54cee41c0","commitTimeStamp":"2015-02-06T06:11:50.5837607Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page588.json","@type":"CatalogPage","commitId":"27f633a7-2956-4462-9636-d5d6a96e9fb4","commitTimeStamp":"2015-02-06T06:32:24.6888922Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page589.json","@type":"CatalogPage","commitId":"2fee17a8-e057-473f-9669-4e768dd6cd94","commitTimeStamp":"2015-02-06T06:44:03.4562789Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page590.json","@type":"CatalogPage","commitId":"24f4e69c-fa98-420f-ab40-7b602eb28941","commitTimeStamp":"2015-02-06T06:58:51.8240113Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page591.json","@type":"CatalogPage","commitId":"ffd04bd6-2015-4ab8-8ef1-c07cca1dbfb1","commitTimeStamp":"2015-02-06T07:13:42.9588539Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page592.json","@type":"CatalogPage","commitId":"65d284c6-4681-4b4c-8d33-3ab01fcf1d9c","commitTimeStamp":"2015-02-06T07:30:26.8237168Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page593.json","@type":"CatalogPage","commitId":"a221dd61-385a-48b1-aef5-f036a1a0856e","commitTimeStamp":"2015-02-06T07:43:41.5072504Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page594.json","@type":"CatalogPage","commitId":"26c85ed3-4fa5-4757-8343-a03b30f1e6f9","commitTimeStamp":"2015-02-06T08:02:55.1561177Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page595.json","@type":"CatalogPage","commitId":"14fcc09d-7eff-4dc4-acff-4da784bc985a","commitTimeStamp":"2015-02-06T08:16:00.1238002Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page596.json","@type":"CatalogPage","commitId":"e489a4f7-81c2-479a-9693-843eef5890d3","commitTimeStamp":"2015-02-06T08:32:22.7059607Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page597.json","@type":"CatalogPage","commitId":"9353b0fe-8276-4180-ac41-6b21a04eb445","commitTimeStamp":"2015-02-06T08:45:11.7379198Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page598.json","@type":"CatalogPage","commitId":"4c3164dd-6b7a-4231-9081-1484f27912b4","commitTimeStamp":"2015-02-06T08:56:37.404425Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page599.json","@type":"CatalogPage","commitId":"8c1cb316-4f07-429d-95d2-b240999b2334","commitTimeStamp":"2015-02-06T09:08:37.2885989Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page600.json","@type":"CatalogPage","commitId":"79a17e71-d50e-457b-87bf-04a8e39c3df6","commitTimeStamp":"2015-02-06T09:24:10.0101227Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page601.json","@type":"CatalogPage","commitId":"585a8817-ef5c-490f-a6a0-ba8222f35ea3","commitTimeStamp":"2015-02-06T09:37:36.0597522Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page602.json","@type":"CatalogPage","commitId":"4db3757e-c7be-4e9f-a20d-5c56fb7074b6","commitTimeStamp":"2015-02-06T09:50:15.0452119Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page603.json","@type":"CatalogPage","commitId":"c0ec166b-73bb-4574-917a-1c7a1d314cca","commitTimeStamp":"2015-02-06T10:02:01.524223Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page604.json","@type":"CatalogPage","commitId":"9d95d038-3111-43a8-98ff-5fc951d5689d","commitTimeStamp":"2015-02-06T10:14:59.680449Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page605.json","@type":"CatalogPage","commitId":"0cc08944-b222-43a1-9764-e1e4566efe79","commitTimeStamp":"2015-02-06T10:26:57.4001165Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page606.json","@type":"CatalogPage","commitId":"6f037b40-e432-4310-8d38-8f8a7285aa48","commitTimeStamp":"2015-02-06T10:39:21.9460914Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page607.json","@type":"CatalogPage","commitId":"dd805940-7cb2-4193-bc58-ec446990899a","commitTimeStamp":"2015-02-06T10:53:08.3167261Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page608.json","@type":"CatalogPage","commitId":"a62ab954-261f-4a41-a740-7f8e6127767b","commitTimeStamp":"2015-02-06T11:07:11.9513369Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page609.json","@type":"CatalogPage","commitId":"130089ed-3bd1-47b2-ba3f-c641f22c9bd1","commitTimeStamp":"2015-02-06T11:22:27.1407662Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page610.json","@type":"CatalogPage","commitId":"0ed2ceaf-d65a-4bd4-add4-e77fc93132b9","commitTimeStamp":"2015-02-06T11:37:12.1329524Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page611.json","@type":"CatalogPage","commitId":"4d03168e-7a39-4777-8964-08982c9d4686","commitTimeStamp":"2015-02-06T11:49:23.9192486Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page612.json","@type":"CatalogPage","commitId":"209e2b8b-e87d-4bf2-a2e8-cc92feb96812","commitTimeStamp":"2015-02-06T12:02:02.8068302Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page613.json","@type":"CatalogPage","commitId":"3c88f469-6e1e-4a96-bc32-988361abfdd2","commitTimeStamp":"2015-02-06T12:16:23.8123675Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page614.json","@type":"CatalogPage","commitId":"1028ea5b-e13b-4474-9472-44230b50a7b8","commitTimeStamp":"2015-02-06T12:32:30.573971Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page615.json","@type":"CatalogPage","commitId":"0e6a6eab-01ac-4eb8-8e49-3b43fbf53b10","commitTimeStamp":"2015-02-06T12:47:23.4767762Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page616.json","@type":"CatalogPage","commitId":"54059edb-a428-42b1-98eb-86495580a061","commitTimeStamp":"2015-02-06T12:59:35.7865284Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page617.json","@type":"CatalogPage","commitId":"b47f54a9-2e95-4976-a5c0-51ebea16ff6b","commitTimeStamp":"2015-02-06T13:14:01.4710444Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page618.json","@type":"CatalogPage","commitId":"13dcf662-3274-4852-a955-5fdfada93924","commitTimeStamp":"2015-02-06T13:32:02.9082044Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page619.json","@type":"CatalogPage","commitId":"500f73ee-9a35-4a68-bb89-8f3d062a4c86","commitTimeStamp":"2015-02-06T13:46:43.6741994Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page620.json","@type":"CatalogPage","commitId":"f5167bf4-5be7-418e-9c0e-3cca728f4396","commitTimeStamp":"2015-02-06T13:59:45.7392316Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page621.json","@type":"CatalogPage","commitId":"0825eaaf-7e8d-4685-a058-014b138f0fe4","commitTimeStamp":"2015-02-06T14:14:02.5462112Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page622.json","@type":"CatalogPage","commitId":"d9f86d68-2047-4a26-a6b2-106063d08715","commitTimeStamp":"2015-02-06T14:31:34.4880451Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page623.json","@type":"CatalogPage","commitId":"c4d9c4bc-b9d2-40a8-9d84-038433330f0d","commitTimeStamp":"2015-02-06T14:46:17.7308643Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page624.json","@type":"CatalogPage","commitId":"7d7cc8b7-2625-446d-9d4b-7b92e2d812c1","commitTimeStamp":"2015-02-06T14:56:34.0495568Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page625.json","@type":"CatalogPage","commitId":"134a160d-d331-48fc-a01e-4a408bdbe222","commitTimeStamp":"2015-02-06T15:07:36.4062778Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page626.json","@type":"CatalogPage","commitId":"1a589684-1542-4cd8-9b1f-980e38f20a03","commitTimeStamp":"2015-02-06T15:22:03.5561877Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page627.json","@type":"CatalogPage","commitId":"e12fb742-eacb-4058-a7a9-d3bb2a702fc7","commitTimeStamp":"2015-02-06T15:34:42.1456612Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page628.json","@type":"CatalogPage","commitId":"c0798ae5-9693-41f3-8718-ae98b7a4e1c9","commitTimeStamp":"2015-02-06T15:47:31.1670348Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page629.json","@type":"CatalogPage","commitId":"ce66253b-d254-448f-8c3a-3f31183634d9","commitTimeStamp":"2015-02-06T16:02:44.2406919Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page630.json","@type":"CatalogPage","commitId":"fbec5edb-49ee-4b8f-a427-38a696a772e1","commitTimeStamp":"2015-02-06T16:20:37.6955816Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page631.json","@type":"CatalogPage","commitId":"1e2e32b3-dfd5-4fd4-8d77-5ed252dc8a31","commitTimeStamp":"2015-02-06T16:33:05.1147921Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page632.json","@type":"CatalogPage","commitId":"4d3bb55f-ba32-4462-8e47-ee9080b0bace","commitTimeStamp":"2015-02-06T16:50:03.4679209Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page633.json","@type":"CatalogPage","commitId":"a9cfc632-6c78-4c10-a382-e6267ab08a42","commitTimeStamp":"2015-02-06T17:02:57.6085262Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page634.json","@type":"CatalogPage","commitId":"4b173dcb-1d57-4444-96bf-5505b46b6166","commitTimeStamp":"2015-02-06T17:15:07.4696551Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page635.json","@type":"CatalogPage","commitId":"d1d37d4c-bee7-48a7-8be9-6b9477509a92","commitTimeStamp":"2015-02-06T17:44:13.0208781Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page636.json","@type":"CatalogPage","commitId":"4049636c-39fe-47ad-a315-875ea518fe5a","commitTimeStamp":"2015-02-06T17:59:18.0352789Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page637.json","@type":"CatalogPage","commitId":"f78a9198-7c26-4691-87a4-ba50eb10e9ed","commitTimeStamp":"2015-02-06T18:20:11.1375788Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page638.json","@type":"CatalogPage","commitId":"c2057d80-0160-49b8-b0aa-065e6433e4b8","commitTimeStamp":"2015-02-06T18:48:29.0083259Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page639.json","@type":"CatalogPage","commitId":"88cd21c6-ed10-4c46-abee-972db0d6c193","commitTimeStamp":"2015-02-06T19:00:30.5610982Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page640.json","@type":"CatalogPage","commitId":"3fa4161b-9138-4faf-8d44-1728d296096d","commitTimeStamp":"2015-02-06T19:27:39.7563011Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page641.json","@type":"CatalogPage","commitId":"5fb6df18-48b6-4021-9ee8-c825672fb9b4","commitTimeStamp":"2015-02-06T19:41:37.3391634Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page642.json","@type":"CatalogPage","commitId":"a6b6c510-c075-4200-878d-a0c91dc3395d","commitTimeStamp":"2015-02-06T19:56:01.0837611Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page643.json","@type":"CatalogPage","commitId":"99bd6594-71b2-4cd4-a122-160291f3df15","commitTimeStamp":"2015-02-06T20:08:36.0412397Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page644.json","@type":"CatalogPage","commitId":"0b10d802-fbe2-4859-a4b0-e7c6b3bafaa8","commitTimeStamp":"2015-02-06T20:21:56.2980553Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page645.json","@type":"CatalogPage","commitId":"68d37c57-b71c-4ee1-8f4b-7937ea8b3bf8","commitTimeStamp":"2015-02-06T20:33:48.6029549Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page646.json","@type":"CatalogPage","commitId":"b456ee4e-6fe0-4dc4-a5a0-c84c723b1701","commitTimeStamp":"2015-02-06T20:50:03.4878842Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page647.json","@type":"CatalogPage","commitId":"53f76eef-fc3d-472f-bd4d-0fd1c554752b","commitTimeStamp":"2015-02-06T21:03:09.5635341Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page648.json","@type":"CatalogPage","commitId":"9f4b736f-4e94-4cfc-9d58-2143c54c2cd9","commitTimeStamp":"2015-02-06T21:31:24.0092948Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page649.json","@type":"CatalogPage","commitId":"2789c579-6194-45a7-9e37-bd1f36bfed87","commitTimeStamp":"2015-02-06T21:48:30.1924073Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page650.json","@type":"CatalogPage","commitId":"40d5a6d4-f3ba-4699-8bbe-234d9de53d67","commitTimeStamp":"2015-02-06T22:07:03.587477Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page651.json","@type":"CatalogPage","commitId":"0d10230f-0635-4fdc-b1e1-16f122880749","commitTimeStamp":"2015-02-06T22:22:52.960358Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page652.json","@type":"CatalogPage","commitId":"7f00d5e4-6524-4525-8bdb-11dde6612442","commitTimeStamp":"2015-02-06T22:38:22.9795855Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page653.json","@type":"CatalogPage","commitId":"1d442e19-1768-46d9-8a38-84ac2db645d6","commitTimeStamp":"2015-02-06T22:56:26.0454256Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page654.json","@type":"CatalogPage","commitId":"3855c53d-cff4-4e18-b67f-3a0a13ee686e","commitTimeStamp":"2015-02-06T23:34:47.7044068Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page655.json","@type":"CatalogPage","commitId":"161d8c62-dd1e-4cb9-beaf-da5606a178c5","commitTimeStamp":"2015-02-06T23:50:43.4299662Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page656.json","@type":"CatalogPage","commitId":"1dbbc0ca-1b2b-4b6a-94e1-f7bb18080c29","commitTimeStamp":"2015-02-07T00:08:20.1067487Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page657.json","@type":"CatalogPage","commitId":"30641d5d-81f5-4669-8920-19e60d2a7e9c","commitTimeStamp":"2015-02-07T00:33:27.9869287Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page658.json","@type":"CatalogPage","commitId":"1ae0eaa8-4e71-4975-8885-32a0911b8e4d","commitTimeStamp":"2015-02-07T00:52:13.8660537Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page659.json","@type":"CatalogPage","commitId":"19ed8289-5974-488a-9836-e398304901b4","commitTimeStamp":"2015-02-07T01:13:30.1981534Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page660.json","@type":"CatalogPage","commitId":"8392b10f-d9a5-43e1-80c7-ed935d2251d9","commitTimeStamp":"2015-02-07T01:35:10.8162936Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page661.json","@type":"CatalogPage","commitId":"6f1d3f10-a674-48e0-a1ad-c0a5b60ec6a1","commitTimeStamp":"2015-02-07T01:51:57.6541317Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page662.json","@type":"CatalogPage","commitId":"d8448573-9588-42d6-95d3-5913ba9946f1","commitTimeStamp":"2015-02-07T02:09:37.6399642Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page663.json","@type":"CatalogPage","commitId":"4acf3c1c-2124-4b0f-94ca-51bcfb5ab722","commitTimeStamp":"2015-02-07T02:24:36.4542151Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page664.json","@type":"CatalogPage","commitId":"bee5b3d8-814a-4ce4-9f92-744f127089e5","commitTimeStamp":"2015-02-07T02:41:58.3712222Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page665.json","@type":"CatalogPage","commitId":"306ce7b0-3848-4ca0-99bc-75d92d236f91","commitTimeStamp":"2015-02-07T02:58:43.2707965Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page666.json","@type":"CatalogPage","commitId":"67b714e9-2faf-45b3-b69e-2eaf3f36b4fb","commitTimeStamp":"2015-02-07T03:14:06.1024739Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page667.json","@type":"CatalogPage","commitId":"d7576f0d-4ddd-48fb-ac16-f845782f8f21","commitTimeStamp":"2015-02-07T03:30:00.6419573Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page668.json","@type":"CatalogPage","commitId":"8f3d4fce-b306-4b8f-a5af-fec7e217c6f1","commitTimeStamp":"2015-02-07T03:48:11.3959753Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page669.json","@type":"CatalogPage","commitId":"49d2a778-b99f-4b87-8515-8961dfcf81cf","commitTimeStamp":"2015-02-07T04:07:52.2904984Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page670.json","@type":"CatalogPage","commitId":"f600d732-cb3c-4677-889d-5d5137736755","commitTimeStamp":"2015-02-07T04:25:24.2951057Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page671.json","@type":"CatalogPage","commitId":"0a9a1b51-aac1-4b59-83dd-e80d05151557","commitTimeStamp":"2015-02-07T04:48:17.3045902Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page672.json","@type":"CatalogPage","commitId":"cfe7a9ee-2436-4773-aac8-bcae04e9cb9b","commitTimeStamp":"2015-02-07T05:08:41.3211155Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page673.json","@type":"CatalogPage","commitId":"67d863d8-a544-43b0-a31a-385d0cad527b","commitTimeStamp":"2015-02-07T05:27:59.4799494Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page674.json","@type":"CatalogPage","commitId":"77cb07cb-76fe-47d9-9fd7-065a04dc8b80","commitTimeStamp":"2015-02-07T05:45:44.9788869Z","count":538},{"@id":"https://api.nuget.org/v3/catalog0/page675.json","@type":"CatalogPage","commitId":"ee37c6bd-aa8f-45fd-ad64-870e79cced30","commitTimeStamp":"2015-02-07T06:06:52.0319936Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page676.json","@type":"CatalogPage","commitId":"44658169-cab5-4e58-a42e-5b2b36558de5","commitTimeStamp":"2015-02-07T06:27:27.9783262Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page677.json","@type":"CatalogPage","commitId":"837acbfc-bd32-445e-bc40-1ed31478d404","commitTimeStamp":"2015-02-07T06:51:17.0222745Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page678.json","@type":"CatalogPage","commitId":"c49757ab-da92-49d6-b405-a0985698fc68","commitTimeStamp":"2015-02-07T07:09:59.9430414Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page679.json","@type":"CatalogPage","commitId":"8fc543b2-4ceb-4ec8-8aa6-686e0a8ad561","commitTimeStamp":"2015-02-07T07:29:26.1296855Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page680.json","@type":"CatalogPage","commitId":"77f3928d-361e-429b-bf16-e8ebd33739de","commitTimeStamp":"2015-02-07T07:48:30.3610127Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page681.json","@type":"CatalogPage","commitId":"0be2ed51-8b94-4e22-a966-1afca10e47ad","commitTimeStamp":"2015-02-07T08:14:20.3986631Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page682.json","@type":"CatalogPage","commitId":"90b7458a-7b79-4b35-9f32-68d0c83888cc","commitTimeStamp":"2015-02-07T08:27:56.8510882Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page683.json","@type":"CatalogPage","commitId":"a1621616-9e84-4394-88ab-b0fb40e36210","commitTimeStamp":"2015-02-07T08:39:20.9003182Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page684.json","@type":"CatalogPage","commitId":"4e384f9f-bb57-4e2b-af6f-9be41f5acfc6","commitTimeStamp":"2015-02-07T08:51:56.1566102Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page685.json","@type":"CatalogPage","commitId":"ad7865d0-f0fc-4998-a0b1-6e8207b869bf","commitTimeStamp":"2015-02-07T09:05:52.4614984Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page686.json","@type":"CatalogPage","commitId":"265846f1-dc2c-46e7-8a43-8a383055b826","commitTimeStamp":"2015-02-07T09:18:54.3087489Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page687.json","@type":"CatalogPage","commitId":"a2886ef1-98e9-48de-90cb-30bb7968c230","commitTimeStamp":"2015-02-07T09:32:34.2718267Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page688.json","@type":"CatalogPage","commitId":"1cc01d14-6257-4af4-a920-30ce02bc1476","commitTimeStamp":"2015-02-07T09:45:27.2594393Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page689.json","@type":"CatalogPage","commitId":"f34d74f8-958f-4e09-b662-026ffd776d40","commitTimeStamp":"2015-02-07T09:56:13.9310275Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page690.json","@type":"CatalogPage","commitId":"0dbe45db-0c33-4ec9-8d09-a8da907638a3","commitTimeStamp":"2015-02-07T10:09:16.0635637Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page691.json","@type":"CatalogPage","commitId":"7e46b99b-fdc6-4f8c-ade6-5346e27b6ecd","commitTimeStamp":"2015-02-07T10:21:14.7358769Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page692.json","@type":"CatalogPage","commitId":"01f33390-e174-41c9-b494-82ebb8be8bc7","commitTimeStamp":"2015-02-07T10:36:51.4020282Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page693.json","@type":"CatalogPage","commitId":"ef328e42-8f25-4401-aaab-5fe685bb44b6","commitTimeStamp":"2015-02-07T10:50:38.5518572Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page694.json","@type":"CatalogPage","commitId":"59dfe3bc-1f10-40bf-bb98-fc41e2578ea9","commitTimeStamp":"2015-02-07T11:03:24.3712107Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page695.json","@type":"CatalogPage","commitId":"e3c4cf3b-c458-4deb-b7df-98fd56692581","commitTimeStamp":"2015-02-07T11:15:41.805569Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page696.json","@type":"CatalogPage","commitId":"a1c9a2a9-c2b2-4d0c-a16d-208427007267","commitTimeStamp":"2015-02-07T11:27:57.8018355Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page697.json","@type":"CatalogPage","commitId":"f5900724-2b45-4449-a9d2-cebab5c754d5","commitTimeStamp":"2015-02-07T11:41:16.394469Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page698.json","@type":"CatalogPage","commitId":"cb947864-eceb-4c2b-95ce-3f1004f63b31","commitTimeStamp":"2015-02-07T11:53:32.4684679Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page699.json","@type":"CatalogPage","commitId":"a2b4cadc-cfd4-4217-becc-91c299ff40b9","commitTimeStamp":"2015-02-07T12:05:26.8409509Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page700.json","@type":"CatalogPage","commitId":"b47f7254-996b-4251-a065-1653db7f93a9","commitTimeStamp":"2015-02-07T12:16:49.8061624Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page701.json","@type":"CatalogPage","commitId":"27a42d8c-7bd7-472e-b02b-99d7ae29c726","commitTimeStamp":"2015-02-07T12:30:10.3657714Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page702.json","@type":"CatalogPage","commitId":"619bb2fb-db43-4784-9671-d5e20900495c","commitTimeStamp":"2015-02-07T12:42:09.1784402Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page703.json","@type":"CatalogPage","commitId":"c3871b31-6b63-42ce-b639-8375091bdcfe","commitTimeStamp":"2015-02-07T12:53:20.5158521Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page704.json","@type":"CatalogPage","commitId":"f8736449-b849-4910-8fa7-88061b8da7e6","commitTimeStamp":"2015-02-07T13:04:41.5720501Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page705.json","@type":"CatalogPage","commitId":"4fd819f0-a17b-4c1a-aac8-ea5f392a8bd2","commitTimeStamp":"2015-02-07T13:16:45.0864441Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page706.json","@type":"CatalogPage","commitId":"adc20961-4d48-4e52-8433-438664bc9ab1","commitTimeStamp":"2015-02-07T13:28:12.5817226Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page707.json","@type":"CatalogPage","commitId":"084dfff9-24e3-47d2-a36f-465aea4a4bb0","commitTimeStamp":"2015-02-07T13:42:08.6747805Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page708.json","@type":"CatalogPage","commitId":"50ffae6f-05ec-485f-a00b-e09adcf5db6b","commitTimeStamp":"2015-02-07T13:55:13.1122118Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page709.json","@type":"CatalogPage","commitId":"ef9bd9af-7148-4d25-be95-de613c2df88f","commitTimeStamp":"2015-02-07T14:07:41.9070289Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page710.json","@type":"CatalogPage","commitId":"f646a356-72ea-43e5-85d2-54124940c3ff","commitTimeStamp":"2015-02-07T14:21:21.6837634Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page711.json","@type":"CatalogPage","commitId":"ec532dbb-61b6-43ed-9e9d-d45652bb66e8","commitTimeStamp":"2015-02-07T14:33:19.3863918Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page712.json","@type":"CatalogPage","commitId":"0fb6bbf4-b802-43d9-85c8-3be72dfaa774","commitTimeStamp":"2015-02-07T14:44:53.4906305Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page713.json","@type":"CatalogPage","commitId":"5da3c63f-2293-476f-8aaa-f39f08034970","commitTimeStamp":"2015-02-07T14:56:40.6353843Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page714.json","@type":"CatalogPage","commitId":"0195921f-e92f-414a-85dd-3adbb12fc8fc","commitTimeStamp":"2015-02-07T15:08:16.9829625Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page715.json","@type":"CatalogPage","commitId":"93b49dfa-308f-4ad2-bea0-0427924bdb2e","commitTimeStamp":"2015-02-07T15:21:37.0497413Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page716.json","@type":"CatalogPage","commitId":"01e3a362-cab9-4d69-aca3-106512e95d4d","commitTimeStamp":"2015-02-07T15:34:31.68034Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page717.json","@type":"CatalogPage","commitId":"96470074-3401-44a4-937e-70ce2af7a3c7","commitTimeStamp":"2015-02-07T15:47:40.7153456Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page718.json","@type":"CatalogPage","commitId":"42dee30b-b717-49e6-b175-429f7ffef601","commitTimeStamp":"2015-02-07T15:59:57.6634984Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page719.json","@type":"CatalogPage","commitId":"85a8c484-6c2a-4314-a3fa-d05aa0ac0450","commitTimeStamp":"2015-02-07T16:11:08.6286977Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page720.json","@type":"CatalogPage","commitId":"13d22d24-5a28-4ec3-9d88-892eefa8e5b1","commitTimeStamp":"2015-02-07T16:24:08.8589597Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page721.json","@type":"CatalogPage","commitId":"0118718a-f138-4a7d-9a41-fbcbe7865469","commitTimeStamp":"2015-02-07T16:37:03.4353601Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page722.json","@type":"CatalogPage","commitId":"42c3f5ef-6c0b-4230-9e73-18faf7b5e55d","commitTimeStamp":"2015-02-07T16:48:47.9385943Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page723.json","@type":"CatalogPage","commitId":"e5e2b7ec-2862-409b-a67e-7f4cc156921b","commitTimeStamp":"2015-02-07T17:02:18.6114683Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page724.json","@type":"CatalogPage","commitId":"2771f7d8-a6c3-498e-a31c-02ca27de16c4","commitTimeStamp":"2015-02-07T17:14:26.7328809Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page725.json","@type":"CatalogPage","commitId":"71f63f92-c528-4e86-a7f6-f5fed45566b0","commitTimeStamp":"2015-02-07T17:27:05.7343841Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page726.json","@type":"CatalogPage","commitId":"3c282f6c-a6c4-495c-ab60-b8459a4a2456","commitTimeStamp":"2015-02-07T17:39:25.1424499Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page727.json","@type":"CatalogPage","commitId":"78addef6-9751-4978-83cf-67093e336f0a","commitTimeStamp":"2015-02-07T17:51:27.1223913Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page728.json","@type":"CatalogPage","commitId":"6d8571c8-378a-4b8a-ae8e-dc3580ed6c93","commitTimeStamp":"2015-02-07T18:04:42.825249Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page729.json","@type":"CatalogPage","commitId":"900796ed-6bf8-4d5e-a177-76c028bf1942","commitTimeStamp":"2015-02-07T18:17:29.3676456Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page730.json","@type":"CatalogPage","commitId":"44e199ae-3230-485b-bbe6-e85724400867","commitTimeStamp":"2015-02-07T18:30:36.7643937Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page731.json","@type":"CatalogPage","commitId":"38cbbece-f92b-4bf5-bcd2-41445e6733fa","commitTimeStamp":"2015-02-07T18:44:05.8031577Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page732.json","@type":"CatalogPage","commitId":"13090076-6400-4f2e-99bb-523d0a3bb58a","commitTimeStamp":"2015-02-07T18:56:52.2447619Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page733.json","@type":"CatalogPage","commitId":"30d919de-d18c-4ca9-8329-96913aaa9be8","commitTimeStamp":"2015-02-07T19:08:48.843098Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page734.json","@type":"CatalogPage","commitId":"4d110db9-bb51-46a3-8661-e039b3ec65a2","commitTimeStamp":"2015-02-07T19:23:28.7209847Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page735.json","@type":"CatalogPage","commitId":"f9c93417-ef6f-49c0-aa83-b0152f9f1bc2","commitTimeStamp":"2015-02-07T19:35:34.133513Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page736.json","@type":"CatalogPage","commitId":"03459f02-8740-43a4-aafb-63b4e7b031a5","commitTimeStamp":"2015-02-07T19:50:13.9789455Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page737.json","@type":"CatalogPage","commitId":"3d965836-cc62-41d9-a362-0d87863dced2","commitTimeStamp":"2015-02-07T19:59:40.3400023Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page738.json","@type":"CatalogPage","commitId":"604c30bb-c6b4-4b66-9d13-d9f1084349dc","commitTimeStamp":"2015-02-07T20:12:38.4005714Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page739.json","@type":"CatalogPage","commitId":"60d6b728-caec-4be5-b813-9e8a5b0c50fb","commitTimeStamp":"2015-02-07T20:24:07.0857637Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page740.json","@type":"CatalogPage","commitId":"54f06d6a-c0be-4164-b56f-5809e6761c4d","commitTimeStamp":"2015-02-07T20:36:01.2637057Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page741.json","@type":"CatalogPage","commitId":"41e32876-2f28-4054-9c5f-de90800c60de","commitTimeStamp":"2015-02-07T20:53:21.3411624Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page742.json","@type":"CatalogPage","commitId":"9cb51462-9c67-4253-9813-e1e049a0b398","commitTimeStamp":"2015-02-07T21:07:07.6591213Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page743.json","@type":"CatalogPage","commitId":"9bfdec88-c9ef-439d-8939-d3fb0d8c93c3","commitTimeStamp":"2015-02-07T21:19:39.467832Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page744.json","@type":"CatalogPage","commitId":"03b0aa8b-f393-4a40-8685-fb227cc68dbf","commitTimeStamp":"2015-02-07T21:31:41.36332Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page745.json","@type":"CatalogPage","commitId":"c94397c2-d18d-4811-868b-eb8c16f6cc30","commitTimeStamp":"2015-02-07T21:46:33.686963Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page746.json","@type":"CatalogPage","commitId":"24a39658-66e6-4b1f-b2c2-7c57e18a68ad","commitTimeStamp":"2015-02-07T22:00:30.0317501Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page747.json","@type":"CatalogPage","commitId":"42facf50-6862-475d-8151-34102a45fc97","commitTimeStamp":"2015-02-07T22:11:34.3766533Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page748.json","@type":"CatalogPage","commitId":"c2ba796c-4555-4528-b6a6-8eb87902210d","commitTimeStamp":"2015-02-07T22:24:35.847751Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page749.json","@type":"CatalogPage","commitId":"056ce7c4-2b2a-4996-9c88-efb49e5e4407","commitTimeStamp":"2015-02-07T22:37:58.3981751Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page750.json","@type":"CatalogPage","commitId":"deb82867-c692-405a-a636-0a8def6f4e8d","commitTimeStamp":"2015-02-07T22:50:51.929027Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page751.json","@type":"CatalogPage","commitId":"10b07739-f711-4e97-b5b6-59037230f53a","commitTimeStamp":"2015-02-07T23:04:39.7834282Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page752.json","@type":"CatalogPage","commitId":"879be5a7-11ec-4ada-b6ae-dc101d0bd712","commitTimeStamp":"2015-02-07T23:17:11.1577444Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page753.json","@type":"CatalogPage","commitId":"4eb725ea-835c-4501-84e1-03042aef9ab4","commitTimeStamp":"2015-02-07T23:32:28.7551086Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page754.json","@type":"CatalogPage","commitId":"d785879c-1aad-4453-8546-d6a16610a034","commitTimeStamp":"2015-02-07T23:48:12.7215342Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page755.json","@type":"CatalogPage","commitId":"46c4efda-156e-41c3-b9d5-01ed566fe880","commitTimeStamp":"2015-02-08T00:05:17.80274Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page756.json","@type":"CatalogPage","commitId":"d51b0a2b-eea5-42f8-b04e-45a3329fc512","commitTimeStamp":"2015-02-08T00:21:15.1045777Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page757.json","@type":"CatalogPage","commitId":"64fa1a7c-7eac-4cbd-aea8-f20913e4e8ae","commitTimeStamp":"2015-02-08T00:43:06.7341457Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page758.json","@type":"CatalogPage","commitId":"51992647-fab1-4695-8eb7-8762fe35002a","commitTimeStamp":"2015-02-08T01:02:38.4877534Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page759.json","@type":"CatalogPage","commitId":"02b45ca1-2650-4b12-ab23-4c0bc96d4b62","commitTimeStamp":"2015-02-08T01:29:11.896261Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page760.json","@type":"CatalogPage","commitId":"aafc42ba-93dc-40b1-bc62-30de79e13d57","commitTimeStamp":"2015-02-08T01:49:11.6851667Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page761.json","@type":"CatalogPage","commitId":"a9a800d6-5618-4207-b250-98435be183a9","commitTimeStamp":"2015-02-08T02:05:18.9153951Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page762.json","@type":"CatalogPage","commitId":"8d7c4c32-acf9-4b56-9bb8-d1e8810a8e9a","commitTimeStamp":"2015-02-08T02:21:01.0356109Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page763.json","@type":"CatalogPage","commitId":"67b6bfed-1322-4971-b56b-96ff64336c67","commitTimeStamp":"2015-02-08T02:37:04.6405693Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page764.json","@type":"CatalogPage","commitId":"8431ee59-6e91-4dc5-88e5-fefd24f2d770","commitTimeStamp":"2015-02-08T02:51:19.781203Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page765.json","@type":"CatalogPage","commitId":"979aef45-5a32-421a-83ca-3db1915abc10","commitTimeStamp":"2015-02-08T03:08:42.3937934Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page766.json","@type":"CatalogPage","commitId":"49ecfe58-a0c5-4014-9d05-a110752da640","commitTimeStamp":"2015-02-08T03:23:42.7693449Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page767.json","@type":"CatalogPage","commitId":"55856e12-688a-4429-9289-9dc11629d06d","commitTimeStamp":"2015-02-08T03:40:31.3576961Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page768.json","@type":"CatalogPage","commitId":"9478764d-a0b9-4d56-ae51-ca21cc47bf8a","commitTimeStamp":"2015-02-08T03:51:07.3890411Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page769.json","@type":"CatalogPage","commitId":"1ca76597-ffdd-466d-be65-d7d704bd2fea","commitTimeStamp":"2015-02-08T04:03:57.4196231Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page770.json","@type":"CatalogPage","commitId":"7245d9c2-5e56-49f1-a25f-7fbba777224d","commitTimeStamp":"2015-02-08T10:57:04.2359153Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page771.json","@type":"CatalogPage","commitId":"443adca5-6c04-44bb-b1cf-a0229ebd622a","commitTimeStamp":"2015-02-09T08:50:41.4564859Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page772.json","@type":"CatalogPage","commitId":"b06b1024-57e8-4251-b49e-789a483de0c6","commitTimeStamp":"2015-02-09T18:51:21.6624732Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page773.json","@type":"CatalogPage","commitId":"e49253ec-43d0-4d0b-8155-a3e1a2f19d19","commitTimeStamp":"2015-02-12T00:23:46.7758898Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page774.json","@type":"CatalogPage","commitId":"e88a4222-8eb6-4f88-881e-3ca174f14293","commitTimeStamp":"2015-02-12T00:32:18.6026804Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page775.json","@type":"CatalogPage","commitId":"49327d64-65d7-4ec0-8d69-0021615ec0b3","commitTimeStamp":"2015-02-12T00:38:59.4147732Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page776.json","@type":"CatalogPage","commitId":"f44aad37-ec8b-4de6-993d-b43d2c41423d","commitTimeStamp":"2015-02-12T00:46:31.2758228Z","count":537},{"@id":"https://api.nuget.org/v3/catalog0/page777.json","@type":"CatalogPage","commitId":"b23282ff-9191-4047-8977-2b34584c16d0","commitTimeStamp":"2015-02-12T01:00:31.9290275Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page778.json","@type":"CatalogPage","commitId":"7acbbe3d-b950-4393-bcbe-164092df8a55","commitTimeStamp":"2015-02-12T15:53:28.4525516Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page779.json","@type":"CatalogPage","commitId":"75f6eb41-d788-47af-bef2-191ed6c4830a","commitTimeStamp":"2015-02-13T09:04:05.2511705Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page780.json","@type":"CatalogPage","commitId":"bb0d8e71-babd-4910-9d16-b3cb381baf3e","commitTimeStamp":"2015-02-13T20:06:57.694701Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page781.json","@type":"CatalogPage","commitId":"508f9dba-9407-491a-bfc4-0530b54e2359","commitTimeStamp":"2015-02-14T22:15:24.1788502Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page782.json","@type":"CatalogPage","commitId":"1ce2adc6-219e-408d-bbfc-4e77e1126962","commitTimeStamp":"2015-02-15T23:48:29.8199426Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page783.json","@type":"CatalogPage","commitId":"019917a5-ed9e-4dab-996a-203abdc5f5e2","commitTimeStamp":"2015-02-16T08:48:12.6995622Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page784.json","@type":"CatalogPage","commitId":"3dd8469b-f9d0-4997-b4a0-f2791389b21e","commitTimeStamp":"2015-02-16T14:36:08.3677223Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page785.json","@type":"CatalogPage","commitId":"e2392e5f-fe2e-4f1e-bab1-476ff2367428","commitTimeStamp":"2015-02-17T12:17:11.6154757Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page786.json","@type":"CatalogPage","commitId":"d72ef02f-52e0-4757-b03a-4deb29d22ac7","commitTimeStamp":"2015-02-18T06:20:04.8419004Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page787.json","@type":"CatalogPage","commitId":"fda3c549-9866-43ea-a5ea-03fee609d0ba","commitTimeStamp":"2015-02-18T20:37:11.6818999Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page788.json","@type":"CatalogPage","commitId":"b1b9a97b-10f4-4357-b3fa-93114d624890","commitTimeStamp":"2015-02-19T14:21:29.6027097Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page789.json","@type":"CatalogPage","commitId":"a8e5a9ba-0a31-4157-ae65-ea5253a7257e","commitTimeStamp":"2015-02-20T02:14:57.4739794Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page790.json","@type":"CatalogPage","commitId":"50f09f1b-1a0c-40b0-b906-53de06057c84","commitTimeStamp":"2015-02-20T10:27:04.6265451Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page791.json","@type":"CatalogPage","commitId":"1e369c8b-10f4-488c-8e9e-7cea4306b40e","commitTimeStamp":"2015-02-21T09:34:36.1390528Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page792.json","@type":"CatalogPage","commitId":"aace5c01-3e3e-41be-8543-5604a9e32a78","commitTimeStamp":"2015-02-22T11:51:39.6418389Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page793.json","@type":"CatalogPage","commitId":"87121a1d-a610-4ba7-a2a9-59223f05c7a4","commitTimeStamp":"2015-02-23T14:58:40.6802592Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page794.json","@type":"CatalogPage","commitId":"2c621e2f-17a0-459c-b5e1-bfe9cbabd4cd","commitTimeStamp":"2015-02-24T07:17:47.9341467Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page795.json","@type":"CatalogPage","commitId":"06825ee7-c67b-4618-b5e1-153123b06b13","commitTimeStamp":"2015-02-24T20:51:21.5770403Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page796.json","@type":"CatalogPage","commitId":"efbf3e74-edb7-46c0-b82e-3d3c057f0f21","commitTimeStamp":"2015-02-25T13:30:37.1308181Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page797.json","@type":"CatalogPage","commitId":"ccec3267-8a97-4276-97e5-77e813f17978","commitTimeStamp":"2015-02-26T08:06:19.6492776Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page798.json","@type":"CatalogPage","commitId":"f6ab8a2c-b668-47c0-a583-21ef908ce63b","commitTimeStamp":"2015-02-26T22:13:40.0892685Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page799.json","@type":"CatalogPage","commitId":"03c379e7-094c-4eb6-b70a-64ae2b22a7c8","commitTimeStamp":"2015-02-27T19:54:18.9116976Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page800.json","@type":"CatalogPage","commitId":"efcd5477-146e-4fa3-a901-ae9611aca6a8","commitTimeStamp":"2015-02-28T21:48:19.8661096Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page801.json","@type":"CatalogPage","commitId":"718435f6-abee-428c-acc6-56fe8d8226df","commitTimeStamp":"2015-03-02T06:04:13.788839Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page802.json","@type":"CatalogPage","commitId":"df1d90cb-bb98-4695-b7e3-e5d81d295fab","commitTimeStamp":"2015-03-02T20:19:07.6994943Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page803.json","@type":"CatalogPage","commitId":"dbf7f651-27db-4fb6-a21c-b755690115db","commitTimeStamp":"2015-03-03T12:41:40.424268Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page804.json","@type":"CatalogPage","commitId":"a6caf105-969e-4d8d-9580-aed588aedada","commitTimeStamp":"2015-03-04T04:02:13.4430613Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page805.json","@type":"CatalogPage","commitId":"19f0d0f5-5e91-464f-af75-976c685b9250","commitTimeStamp":"2015-03-04T15:54:40.5068036Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page806.json","@type":"CatalogPage","commitId":"933ed058-f24f-469e-811f-acaff21f9204","commitTimeStamp":"2015-03-05T10:51:56.7479071Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page807.json","@type":"CatalogPage","commitId":"f17ef4e7-96f9-4670-820f-bf529810e61e","commitTimeStamp":"2015-03-05T19:45:56.2324582Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page808.json","@type":"CatalogPage","commitId":"75f45e40-5ea9-4db9-83a2-a2ec4086dd7e","commitTimeStamp":"2015-03-06T13:00:51.2393732Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page809.json","@type":"CatalogPage","commitId":"6b1c9467-78f9-4613-8dac-201890a3110c","commitTimeStamp":"2015-03-07T09:10:13.2329646Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page810.json","@type":"CatalogPage","commitId":"5235a2e9-6bba-4157-847b-ac48be64a065","commitTimeStamp":"2015-03-08T12:22:30.5205753Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page811.json","@type":"CatalogPage","commitId":"80509e08-45ba-48d9-b607-c156d9eb4286","commitTimeStamp":"2015-03-09T06:24:22.7995798Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page812.json","@type":"CatalogPage","commitId":"d06b2e62-b11c-40b5-b904-0f06c3b841a4","commitTimeStamp":"2015-03-09T22:03:29.2927114Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page813.json","@type":"CatalogPage","commitId":"1169a02a-4597-40ff-b538-a7bfe90dd66f","commitTimeStamp":"2015-03-10T14:03:12.3296446Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page814.json","@type":"CatalogPage","commitId":"e71e6b04-c23a-47d4-9218-f56913b2d50e","commitTimeStamp":"2015-03-11T08:44:46.8063354Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page815.json","@type":"CatalogPage","commitId":"57facfee-affe-43d0-b049-02487323a3ce","commitTimeStamp":"2015-03-11T17:22:04.773717Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page816.json","@type":"CatalogPage","commitId":"531b382d-ad63-4e7a-a11b-45eef2e87384","commitTimeStamp":"2015-03-12T09:30:58.8339912Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page817.json","@type":"CatalogPage","commitId":"d4095372-9c08-4588-9eaa-743e7eff178d","commitTimeStamp":"2015-03-12T19:59:30.7871758Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page818.json","@type":"CatalogPage","commitId":"e4dd30d2-ee4a-46ee-b6bc-eae79b57395f","commitTimeStamp":"2015-03-12T22:11:27.3867975Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page819.json","@type":"CatalogPage","commitId":"6e442762-e695-492e-97a0-96a202d6f835","commitTimeStamp":"2015-03-13T14:04:12.5286959Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page820.json","@type":"CatalogPage","commitId":"0fbb68bf-5828-4b56-851e-4cddf85224c7","commitTimeStamp":"2015-03-14T08:38:01.9461551Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page821.json","@type":"CatalogPage","commitId":"7f04715f-5235-4e47-b3d3-6c70357d7ed5","commitTimeStamp":"2015-03-14T20:05:28.7922919Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page822.json","@type":"CatalogPage","commitId":"9b0b309e-5708-48b8-8b9b-0f5f516a8ac2","commitTimeStamp":"2015-03-15T17:32:35.0540057Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page823.json","@type":"CatalogPage","commitId":"18d03c94-bed3-4a6d-8f67-8f55ef12dca3","commitTimeStamp":"2015-03-16T16:28:14.8025101Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page824.json","@type":"CatalogPage","commitId":"dd64cb5a-2374-412c-8b56-3b3b6dc32a46","commitTimeStamp":"2015-03-17T11:49:48.1611956Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page825.json","@type":"CatalogPage","commitId":"d6ed7e0b-89f0-40b9-900d-351b72fc2b53","commitTimeStamp":"2015-03-17T23:38:21.1730847Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page826.json","@type":"CatalogPage","commitId":"061eaa8f-c119-4031-9d38-dfe66df51a38","commitTimeStamp":"2015-03-18T23:48:27.5493432Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page827.json","@type":"CatalogPage","commitId":"6fadb962-ae06-4b07-8866-7c61198e6118","commitTimeStamp":"2015-03-19T13:56:04.1787138Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page828.json","@type":"CatalogPage","commitId":"1cd40cf0-c617-4339-8ae0-6f1fe25f0c0d","commitTimeStamp":"2015-03-20T11:22:53.3944946Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page829.json","@type":"CatalogPage","commitId":"87dcc656-54c2-4f9a-bce9-c0b0354996c8","commitTimeStamp":"2015-03-20T23:05:40.4975427Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page830.json","@type":"CatalogPage","commitId":"6879207d-59b5-418d-8d16-439ecb5049ba","commitTimeStamp":"2015-03-22T01:12:29.2955279Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page831.json","@type":"CatalogPage","commitId":"96051529-8fe5-4a02-920c-db71344bb0bc","commitTimeStamp":"2015-03-23T06:21:02.7845582Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page832.json","@type":"CatalogPage","commitId":"550b948b-4742-4c83-acfc-715f65559ea5","commitTimeStamp":"2015-03-23T17:18:52.7456559Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page833.json","@type":"CatalogPage","commitId":"e9a417d0-dd35-4fdc-8713-21337e1116f1","commitTimeStamp":"2015-03-24T09:37:54.1645552Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page834.json","@type":"CatalogPage","commitId":"ceb47156-8832-4de2-9a20-a849cfb72b49","commitTimeStamp":"2015-03-25T00:43:32.6724301Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page835.json","@type":"CatalogPage","commitId":"5ed7ce16-6641-4256-ba33-8553cc0e6f96","commitTimeStamp":"2015-03-25T17:41:11.7577209Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page836.json","@type":"CatalogPage","commitId":"d0b143f9-586f-4754-b50e-14cc9051ff55","commitTimeStamp":"2015-03-26T06:12:46.4500399Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page837.json","@type":"CatalogPage","commitId":"c53a4c79-6ea0-4885-9108-d432e34db286","commitTimeStamp":"2015-03-26T22:27:42.0855681Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page838.json","@type":"CatalogPage","commitId":"592ecd70-05f1-4954-8258-87066db76d2f","commitTimeStamp":"2015-03-27T10:25:46.3896291Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page839.json","@type":"CatalogPage","commitId":"d6414063-fa2c-4181-b0ea-c1d7c50de14c","commitTimeStamp":"2015-03-28T03:19:58.3628752Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page840.json","@type":"CatalogPage","commitId":"dffeee68-2d2b-4750-b673-58f9601ed053","commitTimeStamp":"2015-03-29T09:49:23.6881707Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page841.json","@type":"CatalogPage","commitId":"c6604b38-3d43-4251-81c9-eb90bef73487","commitTimeStamp":"2015-03-30T09:22:07.6511292Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page842.json","@type":"CatalogPage","commitId":"95769c6f-cd7a-4ada-98a7-64cca009a9d5","commitTimeStamp":"2015-03-31T04:59:23.8990079Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page843.json","@type":"CatalogPage","commitId":"b573b89a-0ab4-46b6-95ea-c2116e1c6fe1","commitTimeStamp":"2015-03-31T20:43:38.8039526Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page844.json","@type":"CatalogPage","commitId":"23186e04-b405-4c37-be5a-1539387ea263","commitTimeStamp":"2015-04-01T13:04:03.4431089Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page845.json","@type":"CatalogPage","commitId":"956b0360-a505-4de5-91cb-6e6789d3c2dd","commitTimeStamp":"2015-04-01T22:27:06.4676986Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page846.json","@type":"CatalogPage","commitId":"068a2781-2b4b-4192-9a6f-e1f5dc573dbf","commitTimeStamp":"2015-04-02T17:51:23.2319756Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page847.json","@type":"CatalogPage","commitId":"4ec8897c-d8e0-49b2-8668-9464f0062f02","commitTimeStamp":"2015-04-03T15:34:02.0698496Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page848.json","@type":"CatalogPage","commitId":"54771cb9-3214-4afd-bc10-675bf773dcc8","commitTimeStamp":"2015-04-04T14:49:25.445208Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page849.json","@type":"CatalogPage","commitId":"4a2f6659-8180-4694-8453-41177a5f7ede","commitTimeStamp":"2015-04-05T22:09:34.158241Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page850.json","@type":"CatalogPage","commitId":"9f6c9889-b588-47d5-a4dc-0e0d70138249","commitTimeStamp":"2015-04-06T20:28:52.1104075Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page851.json","@type":"CatalogPage","commitId":"0d232e8f-80de-41c9-a483-36589034df08","commitTimeStamp":"2015-04-07T11:57:59.0803479Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page852.json","@type":"CatalogPage","commitId":"fd0ec229-8eb8-4b7b-976b-67100021ef5e","commitTimeStamp":"2015-04-07T17:28:37.6513485Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page853.json","@type":"CatalogPage","commitId":"fa9d30f5-dce8-468b-bd72-2abc15ad5144","commitTimeStamp":"2015-04-07T18:31:47.5379056Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page854.json","@type":"CatalogPage","commitId":"b528a29f-a9c7-498d-be9d-3912f4188b20","commitTimeStamp":"2015-04-08T06:06:29.8805651Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page855.json","@type":"CatalogPage","commitId":"d7fb97d6-b20c-498a-b727-cd6e8d269703","commitTimeStamp":"2015-04-09T16:33:04.2865455Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page856.json","@type":"CatalogPage","commitId":"52046680-3e3c-4149-95ad-5ea65b47c608","commitTimeStamp":"2015-04-09T16:41:12.5838526Z","count":536},{"@id":"https://api.nuget.org/v3/catalog0/page857.json","@type":"CatalogPage","commitId":"fdf510aa-04af-4081-9397-d5a82a76b652","commitTimeStamp":"2015-04-10T09:57:19.4473347Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page858.json","@type":"CatalogPage","commitId":"5fffe202-31dc-433f-b391-405801722068","commitTimeStamp":"2015-04-11T06:10:35.389418Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page859.json","@type":"CatalogPage","commitId":"8738166f-1a16-4a2a-ae7e-97b3b7f58654","commitTimeStamp":"2015-04-12T12:59:13.8857324Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page860.json","@type":"CatalogPage","commitId":"3cd2807c-715d-4c05-9d5f-596356715292","commitTimeStamp":"2015-04-13T14:00:59.1166504Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page861.json","@type":"CatalogPage","commitId":"19a898cb-cf25-40d3-8565-97f17c231c9e","commitTimeStamp":"2015-04-14T08:11:57.5174631Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page862.json","@type":"CatalogPage","commitId":"5a45af23-4ec1-42c9-9bb6-9c6d0d01c222","commitTimeStamp":"2015-04-14T21:11:28.8214302Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page863.json","@type":"CatalogPage","commitId":"8626bb4d-2947-43d7-a1da-e41247f82b60","commitTimeStamp":"2015-04-16T17:34:01.6642929Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page864.json","@type":"CatalogPage","commitId":"4f2ea2b3-87df-4d7c-8b2f-2568fea83b13","commitTimeStamp":"2015-04-16T17:41:32.6846301Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page865.json","@type":"CatalogPage","commitId":"6b2c7321-79f2-4f51-85c5-09a19bacc2c9","commitTimeStamp":"2015-04-17T02:24:38.7971038Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page866.json","@type":"CatalogPage","commitId":"324cb2db-aafd-4781-bffb-4d895585ab95","commitTimeStamp":"2015-04-17T14:24:42.2241646Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page867.json","@type":"CatalogPage","commitId":"b4eb4ad3-f0ae-455a-a4b0-e8812227cc54","commitTimeStamp":"2015-04-17T23:10:44.9757058Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page868.json","@type":"CatalogPage","commitId":"0710b4c0-e32a-40a4-9269-baefa9d1f3f3","commitTimeStamp":"2015-04-18T16:36:03.6679883Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page869.json","@type":"CatalogPage","commitId":"4d928b50-5a09-4c04-9fb4-96aee74b925a","commitTimeStamp":"2015-04-19T18:08:51.4476734Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page870.json","@type":"CatalogPage","commitId":"6afa35b6-981e-4827-83ef-1373411850e2","commitTimeStamp":"2015-04-20T13:09:51.631163Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page871.json","@type":"CatalogPage","commitId":"41e1469a-1aad-4a0c-8aa9-67fcf8028f0c","commitTimeStamp":"2015-04-21T18:51:29.4369391Z","count":533},{"@id":"https://api.nuget.org/v3/catalog0/page872.json","@type":"CatalogPage","commitId":"b3be01ff-a501-4156-a557-4ba5be5f4524","commitTimeStamp":"2015-04-21T18:56:53.2160891Z","count":541},{"@id":"https://api.nuget.org/v3/catalog0/page873.json","@type":"CatalogPage","commitId":"9f65b6ce-e195-404b-937e-16e92be10ae6","commitTimeStamp":"2015-04-21T22:17:47.2692294Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page874.json","@type":"CatalogPage","commitId":"3df17602-63a8-4573-8850-c23cb6a2fdcc","commitTimeStamp":"2015-04-22T11:08:52.9958229Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page875.json","@type":"CatalogPage","commitId":"dd132306-ca89-454d-83cc-af5f12012559","commitTimeStamp":"2015-04-23T01:01:07.5413463Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page876.json","@type":"CatalogPage","commitId":"fadbd75b-0df8-4078-a5a9-a4fba749a350","commitTimeStamp":"2015-04-23T15:46:42.3688925Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page877.json","@type":"CatalogPage","commitId":"4e3a75fc-477f-4ccd-a30d-e862ac152f9e","commitTimeStamp":"2015-04-24T10:11:10.6519814Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page878.json","@type":"CatalogPage","commitId":"ab49a92f-fdc6-47e7-a605-869449f103fd","commitTimeStamp":"2015-04-27T21:07:22.5051697Z","count":541},{"@id":"https://api.nuget.org/v3/catalog0/page879.json","@type":"CatalogPage","commitId":"dfa780ce-489f-4fdb-9f21-1a56bc8451fa","commitTimeStamp":"2015-04-27T21:13:37.3122763Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page880.json","@type":"CatalogPage","commitId":"c01ee9f4-745b-4e8b-82f8-ddddfbd6ef84","commitTimeStamp":"2015-04-27T21:19:15.548057Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page881.json","@type":"CatalogPage","commitId":"f2ae4139-1399-4965-82e1-9c5769f97d2c","commitTimeStamp":"2015-04-27T21:29:21.768306Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page882.json","@type":"CatalogPage","commitId":"d7551a8f-6b38-4be1-8c7b-a2ea0697cd07","commitTimeStamp":"2015-04-27T21:36:39.8000977Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page883.json","@type":"CatalogPage","commitId":"e237cb67-e210-4f93-a36c-ac53071b3903","commitTimeStamp":"2015-04-28T14:20:05.0567974Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page884.json","@type":"CatalogPage","commitId":"a7b1d3ce-4f21-4207-8d7b-182277917240","commitTimeStamp":"2015-04-29T03:47:43.4115949Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page885.json","@type":"CatalogPage","commitId":"3f434851-bb65-44e8-b4bd-c7eb3ada1f12","commitTimeStamp":"2015-04-29T14:10:20.2467981Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page886.json","@type":"CatalogPage","commitId":"0b04be33-3def-40a7-82f2-ba118023024f","commitTimeStamp":"2015-04-29T20:58:05.0871517Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page887.json","@type":"CatalogPage","commitId":"a82853e9-ef18-4665-a770-6f50e17679b5","commitTimeStamp":"2015-04-30T18:18:37.9307986Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page888.json","@type":"CatalogPage","commitId":"d20a4958-79e5-45d9-9d76-313daa841c65","commitTimeStamp":"2015-05-01T07:50:44.18541Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page889.json","@type":"CatalogPage","commitId":"ec5c82ba-9ff2-4812-8690-5e0519128228","commitTimeStamp":"2015-05-01T20:18:04.3684275Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page890.json","@type":"CatalogPage","commitId":"f25a1d82-cf19-46dd-a9c1-fd7c565cbac3","commitTimeStamp":"2015-05-04T16:39:59.2714915Z","count":541},{"@id":"https://api.nuget.org/v3/catalog0/page891.json","@type":"CatalogPage","commitId":"436eab4e-85ee-4a0a-9bc5-401f8b29740a","commitTimeStamp":"2015-05-04T16:48:00.4721353Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page892.json","@type":"CatalogPage","commitId":"b88c8808-2c6e-4510-b417-573c99c72a98","commitTimeStamp":"2015-05-04T17:09:44.4948942Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page893.json","@type":"CatalogPage","commitId":"d6679758-8c7a-4ecf-bdfb-682f67d7545c","commitTimeStamp":"2015-05-04T21:19:17.5216804Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page894.json","@type":"CatalogPage","commitId":"65a6b41a-3a38-48a7-850f-0843c909137a","commitTimeStamp":"2015-05-05T18:01:59.5795726Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page895.json","@type":"CatalogPage","commitId":"8eae0e30-fe6c-42b4-870b-a0db73645d21","commitTimeStamp":"2015-05-06T17:43:46.6100714Z","count":538},{"@id":"https://api.nuget.org/v3/catalog0/page896.json","@type":"CatalogPage","commitId":"d54bc2ab-e685-4796-ab50-e4340f33b7cf","commitTimeStamp":"2015-05-07T05:59:23.5630806Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page897.json","@type":"CatalogPage","commitId":"b42451b8-c3c9-4652-9fd5-7b731a69968b","commitTimeStamp":"2015-05-07T18:51:15.1228904Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page898.json","@type":"CatalogPage","commitId":"6b0d7092-ad6a-4b01-ba3c-7e526af166f4","commitTimeStamp":"2015-05-08T14:09:21.7057467Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page899.json","@type":"CatalogPage","commitId":"9b272a1a-b63c-42ef-a804-ed520be33991","commitTimeStamp":"2015-05-09T23:30:08.9792606Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page900.json","@type":"CatalogPage","commitId":"8de63ecd-4355-4128-9551-3d6b89c96dae","commitTimeStamp":"2015-05-11T14:09:57.4042462Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page901.json","@type":"CatalogPage","commitId":"808daacc-d29e-4511-bb15-b2b3a6322b00","commitTimeStamp":"2015-05-12T12:10:06.1305419Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page902.json","@type":"CatalogPage","commitId":"ce7e4e4a-a554-4a70-8fca-a492cc1b7158","commitTimeStamp":"2015-05-12T22:32:42.3743535Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page903.json","@type":"CatalogPage","commitId":"b96171ca-46c3-4965-90bc-d9c657122a6c","commitTimeStamp":"2015-05-13T16:37:49.3533194Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page904.json","@type":"CatalogPage","commitId":"e0c7c6c4-c1ca-40ea-9de8-087266b0b029","commitTimeStamp":"2015-05-14T12:06:26.7108685Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page905.json","@type":"CatalogPage","commitId":"dbec35d5-814f-45a8-945f-47e3bd95cccc","commitTimeStamp":"2015-05-15T05:31:12.2785706Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page906.json","@type":"CatalogPage","commitId":"8efbcdd3-e670-4774-8b86-39b3848feda3","commitTimeStamp":"2015-05-15T15:23:07.4242738Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page907.json","@type":"CatalogPage","commitId":"8e78adcd-98eb-4549-a8e6-e9f441c6c175","commitTimeStamp":"2015-05-16T09:53:14.7157316Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page908.json","@type":"CatalogPage","commitId":"7fab05c8-2d10-4d29-af20-4138ece25c01","commitTimeStamp":"2015-05-17T03:14:27.3160184Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page909.json","@type":"CatalogPage","commitId":"54de8112-c6d4-4dab-a50c-b08d100163c4","commitTimeStamp":"2015-05-18T05:22:38.9745624Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page910.json","@type":"CatalogPage","commitId":"fb068daf-48b8-448d-8c15-5c5cd1a492f9","commitTimeStamp":"2015-05-18T15:29:18.8237836Z","count":552},{"@id":"https://api.nuget.org/v3/catalog0/page911.json","@type":"CatalogPage","commitId":"a0bdba81-0170-4762-878e-cc5e40a9722c","commitTimeStamp":"2015-05-19T17:55:48.9862115Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page912.json","@type":"CatalogPage","commitId":"02c8ac73-dec9-499b-85cf-9c8d8e6b759c","commitTimeStamp":"2015-05-20T06:42:12.2445542Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page913.json","@type":"CatalogPage","commitId":"948c8268-71e3-4ed0-a6e3-6a0bd2d67221","commitTimeStamp":"2015-05-20T18:25:48.8375424Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page914.json","@type":"CatalogPage","commitId":"e999fc44-2213-411e-8bad-e6429685d440","commitTimeStamp":"2015-05-21T15:25:34.4322378Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page915.json","@type":"CatalogPage","commitId":"188458ba-8b6f-4a31-a195-674951600e5b","commitTimeStamp":"2015-05-22T08:10:55.9734081Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page916.json","@type":"CatalogPage","commitId":"bd4801f5-b78e-4922-ba44-f2fae418d320","commitTimeStamp":"2015-05-22T20:12:19.1986688Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page917.json","@type":"CatalogPage","commitId":"23cfd57c-e17d-4a9d-8c1f-d9e7e35a189e","commitTimeStamp":"2015-05-23T23:24:38.4651546Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page918.json","@type":"CatalogPage","commitId":"92c593fd-7942-44fe-a310-0752d46a95bb","commitTimeStamp":"2015-05-25T03:49:35.495416Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page919.json","@type":"CatalogPage","commitId":"85c0c2e7-b2d1-42c9-abad-6c4fc1f10a72","commitTimeStamp":"2015-05-26T03:28:37.4826522Z","count":535},{"@id":"https://api.nuget.org/v3/catalog0/page920.json","@type":"CatalogPage","commitId":"40c57eb8-3381-4530-874c-591dc6248b01","commitTimeStamp":"2015-05-26T13:35:49.5578989Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page921.json","@type":"CatalogPage","commitId":"645d17f9-19e7-455d-8ef9-614b0c177e7f","commitTimeStamp":"2015-05-27T04:12:34.899437Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page922.json","@type":"CatalogPage","commitId":"5ed476b4-a854-4731-83b0-6456adbb78ab","commitTimeStamp":"2015-05-27T21:58:38.6795215Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page923.json","@type":"CatalogPage","commitId":"c810ed31-a3ab-47d5-bbb0-301f58b9f2de","commitTimeStamp":"2015-05-28T15:20:53.9880476Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page924.json","@type":"CatalogPage","commitId":"f6968977-af34-4983-bc55-77487d35b937","commitTimeStamp":"2015-05-29T08:41:28.7955901Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page925.json","@type":"CatalogPage","commitId":"dae8a469-a99f-4ed7-a653-3c3d543f940d","commitTimeStamp":"2015-05-29T17:55:21.9246572Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page926.json","@type":"CatalogPage","commitId":"7ac449e8-c83b-4af8-bdaa-dc20ead7b2b7","commitTimeStamp":"2015-05-30T16:59:18.7085257Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page927.json","@type":"CatalogPage","commitId":"72b8a90a-8cb5-4424-839d-540b8b2924c7","commitTimeStamp":"2015-05-31T08:50:16.9616097Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page928.json","@type":"CatalogPage","commitId":"125da764-e802-48b4-ba38-b0cba87a6465","commitTimeStamp":"2015-05-31T15:40:36.28128Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page929.json","@type":"CatalogPage","commitId":"d1873068-f12c-41a3-8019-ca91db42344a","commitTimeStamp":"2015-06-01T05:59:13.5912663Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page930.json","@type":"CatalogPage","commitId":"6450449d-82d1-4cb6-a43f-ff31b768d4df","commitTimeStamp":"2015-06-01T12:47:16.098618Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page931.json","@type":"CatalogPage","commitId":"0dca9d15-7d9e-4533-a762-c71f8566c604","commitTimeStamp":"2015-06-01T16:52:44.9946807Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page932.json","@type":"CatalogPage","commitId":"5d248fc9-ad02-4a0c-b8b7-bfc8dc45aea2","commitTimeStamp":"2015-06-02T09:15:22.354313Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page933.json","@type":"CatalogPage","commitId":"26ef12b3-588c-4109-aa52-5cf53f13cb63","commitTimeStamp":"2015-06-03T01:47:03.8582037Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page934.json","@type":"CatalogPage","commitId":"7bbada91-d41a-46db-ad37-3215b2d760d8","commitTimeStamp":"2015-06-03T19:12:47.6002671Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page935.json","@type":"CatalogPage","commitId":"c3979399-b4d4-47b2-9fa5-0ce5e5a8e1d1","commitTimeStamp":"2015-06-04T11:57:52.0414898Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page936.json","@type":"CatalogPage","commitId":"169a7412-3f60-4afd-8f0c-4451f4b72541","commitTimeStamp":"2015-06-04T21:49:22.0387124Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page937.json","@type":"CatalogPage","commitId":"d8be5455-bb0b-44e1-b65a-f5b48f71316f","commitTimeStamp":"2015-06-05T17:02:13.5699427Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page938.json","@type":"CatalogPage","commitId":"ede48cb0-5e53-42f9-8d12-81270311cc15","commitTimeStamp":"2015-06-07T05:16:28.2029028Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page939.json","@type":"CatalogPage","commitId":"85d39bde-e6a1-4976-a570-004681e90399","commitTimeStamp":"2015-06-08T05:57:06.9473534Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page940.json","@type":"CatalogPage","commitId":"9fe865d8-786f-4b36-b09c-5746cd67a81b","commitTimeStamp":"2015-06-08T20:43:52.3358062Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page941.json","@type":"CatalogPage","commitId":"544d3387-b401-4484-8b32-a8aae671f536","commitTimeStamp":"2015-06-09T11:42:30.2787225Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page942.json","@type":"CatalogPage","commitId":"f41733ee-748f-47ac-97bb-43e66c13f851","commitTimeStamp":"2015-06-09T23:33:54.0545116Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page943.json","@type":"CatalogPage","commitId":"42d5a883-3683-459b-b213-1c8e41bf42b2","commitTimeStamp":"2015-06-10T16:19:01.176625Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page944.json","@type":"CatalogPage","commitId":"269c6796-1df7-4ee9-996e-06be37d344c7","commitTimeStamp":"2015-06-11T13:54:38.5424039Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page945.json","@type":"CatalogPage","commitId":"b05a5b1d-52cb-4d14-880c-5fbff2d52a68","commitTimeStamp":"2015-06-12T11:25:40.1920012Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page946.json","@type":"CatalogPage","commitId":"a63764c6-e069-4e8c-b50a-c3f278e70842","commitTimeStamp":"2015-06-13T07:07:41.4475426Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page947.json","@type":"CatalogPage","commitId":"d98deb21-8cd8-4ae2-bca9-790c16b7eb91","commitTimeStamp":"2015-06-15T00:35:23.131463Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page948.json","@type":"CatalogPage","commitId":"176a68d9-7f9c-48da-a905-f51b3dfd4842","commitTimeStamp":"2015-06-15T18:45:56.1732349Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page949.json","@type":"CatalogPage","commitId":"19900cdd-ef21-4c73-aaab-f216e5c67019","commitTimeStamp":"2015-06-16T17:17:39.5457755Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page950.json","@type":"CatalogPage","commitId":"e11639ed-8c47-4b99-9599-fbce996483d5","commitTimeStamp":"2015-06-17T12:23:57.6463757Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page951.json","@type":"CatalogPage","commitId":"b48ec80c-da0d-41be-9004-12811617b704","commitTimeStamp":"2015-06-18T00:12:12.0058285Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page952.json","@type":"CatalogPage","commitId":"5a1c0b0b-2504-4b57-a243-23e9827ab9e1","commitTimeStamp":"2015-06-18T19:40:57.1346388Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page953.json","@type":"CatalogPage","commitId":"764c5b44-8592-4472-9c46-e67907527f44","commitTimeStamp":"2015-06-19T16:48:09.8492028Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page954.json","@type":"CatalogPage","commitId":"50fe6d33-b2e8-40e0-8bf4-16b027be867d","commitTimeStamp":"2015-06-20T23:24:04.3037601Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page955.json","@type":"CatalogPage","commitId":"7349e7b6-3566-40c6-bb31-6bf5aeb13ad1","commitTimeStamp":"2015-06-22T08:38:45.2340958Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page956.json","@type":"CatalogPage","commitId":"6dd6654f-19b2-46a1-9e47-4cf667a9c38f","commitTimeStamp":"2015-06-22T21:34:21.6184789Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page957.json","@type":"CatalogPage","commitId":"dc58194c-e26e-4185-b68e-e7cfb9923261","commitTimeStamp":"2015-06-23T10:44:53.59165Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page958.json","@type":"CatalogPage","commitId":"18016642-3b9e-4845-9483-3eb4ea720743","commitTimeStamp":"2015-06-24T03:51:55.1029299Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page959.json","@type":"CatalogPage","commitId":"e50d48f3-f62c-4dcc-9b26-3a24a7815094","commitTimeStamp":"2015-06-24T17:12:58.2112336Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page960.json","@type":"CatalogPage","commitId":"dfdf94f8-7f52-4cdc-8fc6-4c2a8c2e1744","commitTimeStamp":"2015-06-25T14:23:26.9846666Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page961.json","@type":"CatalogPage","commitId":"a2c58cb3-2fb9-4b22-8136-56b11f83f3b2","commitTimeStamp":"2015-06-26T08:23:17.0301672Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page962.json","@type":"CatalogPage","commitId":"6e79ee9a-61c7-4262-8d4b-fe7657c89492","commitTimeStamp":"2015-06-27T09:15:30.8099777Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page963.json","@type":"CatalogPage","commitId":"b0292c38-0d4b-4dc5-a496-bd29efb6bd7a","commitTimeStamp":"2015-06-28T21:01:20.7774358Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page964.json","@type":"CatalogPage","commitId":"63db1010-9ede-4a88-b274-ad0b61db2199","commitTimeStamp":"2015-06-29T18:37:42.5998606Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page965.json","@type":"CatalogPage","commitId":"fc2f9382-48c1-4262-b6fd-aeeb8db68ef6","commitTimeStamp":"2015-06-30T13:46:20.1319067Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page966.json","@type":"CatalogPage","commitId":"29406cc1-7351-4d6e-b8d7-abf1f4ff1c49","commitTimeStamp":"2015-06-30T20:53:13.1842241Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page967.json","@type":"CatalogPage","commitId":"eca815e6-8103-42e5-a326-6ef43a19a6cf","commitTimeStamp":"2015-07-01T16:49:46.3942473Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page968.json","@type":"CatalogPage","commitId":"39e97f21-0e7c-49e1-95cf-b7a78cc73b9f","commitTimeStamp":"2015-07-02T05:19:00.3796547Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page969.json","@type":"CatalogPage","commitId":"4516073c-e681-4a98-bf0c-ef67ef875d20","commitTimeStamp":"2015-07-02T17:30:22.2004858Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page970.json","@type":"CatalogPage","commitId":"f8368245-0740-429f-818c-1c67b785c300","commitTimeStamp":"2015-07-03T09:53:40.0789951Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page971.json","@type":"CatalogPage","commitId":"6f7483f2-6047-4da2-bcbd-0ceb33ea5128","commitTimeStamp":"2015-07-04T12:33:29.263643Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page972.json","@type":"CatalogPage","commitId":"35826232-752d-4b6c-bd07-b9d65fa8970a","commitTimeStamp":"2015-07-06T00:55:31.5360132Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page973.json","@type":"CatalogPage","commitId":"850042aa-3432-4971-ae86-dabd91886571","commitTimeStamp":"2015-07-06T17:52:00.1250889Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page974.json","@type":"CatalogPage","commitId":"9300a919-fd28-4563-af55-0428d86380a0","commitTimeStamp":"2015-07-07T11:21:59.3186139Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page975.json","@type":"CatalogPage","commitId":"32152225-918e-498f-b732-e5d4967f8956","commitTimeStamp":"2015-07-07T23:49:10.9432856Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page976.json","@type":"CatalogPage","commitId":"e9cefa4b-70b9-4c6b-ae7e-e17debb86e67","commitTimeStamp":"2015-07-08T15:01:27.7482223Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page977.json","@type":"CatalogPage","commitId":"79c59cc6-b000-4990-bccb-692c9b6966bc","commitTimeStamp":"2015-07-09T12:58:59.9945004Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page978.json","@type":"CatalogPage","commitId":"bd9635b3-fbe4-4115-a701-7bf63e9c7e1f","commitTimeStamp":"2015-07-10T03:55:59.1243203Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page979.json","@type":"CatalogPage","commitId":"1df9a979-fb88-40c5-a97f-158865c2c45d","commitTimeStamp":"2015-07-10T21:32:13.7487276Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page980.json","@type":"CatalogPage","commitId":"69b9141d-125e-4ac4-b24c-3d0afe1c6d2c","commitTimeStamp":"2015-07-11T20:57:03.8595568Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page981.json","@type":"CatalogPage","commitId":"7d69ff60-be6f-40dc-96c5-a3b030b3c66c","commitTimeStamp":"2015-07-12T18:18:25.5430436Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page982.json","@type":"CatalogPage","commitId":"b73ed3e1-751d-4c7f-b8ed-7c19fd0b37aa","commitTimeStamp":"2015-07-13T10:54:46.7059886Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page983.json","@type":"CatalogPage","commitId":"91b15051-b946-4f1a-aa15-b50e82dfd9a1","commitTimeStamp":"2015-07-13T20:16:12.4053088Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page984.json","@type":"CatalogPage","commitId":"f897509a-3284-4ff9-b116-79ec441837d6","commitTimeStamp":"2015-07-14T09:08:45.3683659Z","count":552},{"@id":"https://api.nuget.org/v3/catalog0/page985.json","@type":"CatalogPage","commitId":"d1d05d5f-956d-4d4d-80b7-aebbea16f185","commitTimeStamp":"2015-07-14T16:22:55.4720271Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page986.json","@type":"CatalogPage","commitId":"70cae255-c636-4ff8-954a-0ac19760dc39","commitTimeStamp":"2015-07-15T08:39:14.8709587Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page987.json","@type":"CatalogPage","commitId":"48771a92-292f-4200-8729-a83779b3435b","commitTimeStamp":"2015-07-15T12:47:27.9312788Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page988.json","@type":"CatalogPage","commitId":"5ad8fbee-7e48-49b6-b061-4d59b26106d3","commitTimeStamp":"2015-07-16T05:35:54.3838095Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page989.json","@type":"CatalogPage","commitId":"30c7363f-3837-4386-a40e-6f881b5d39a4","commitTimeStamp":"2015-07-16T16:27:42.4562876Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page990.json","@type":"CatalogPage","commitId":"96e1f984-5723-453c-ba93-56ba10b58fa9","commitTimeStamp":"2015-07-17T02:24:46.9029404Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page991.json","@type":"CatalogPage","commitId":"108b4b7f-a09e-4f95-9945-26cb790cfd3f","commitTimeStamp":"2015-07-17T12:37:59.0573928Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page992.json","@type":"CatalogPage","commitId":"6f062eda-8b0b-45c3-b0d0-ec34bdaf9e4b","commitTimeStamp":"2015-07-18T01:33:55.4523764Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page993.json","@type":"CatalogPage","commitId":"91d1e57d-3616-4b19-ab46-e326754ea5a3","commitTimeStamp":"2015-07-18T12:58:09.8473034Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page994.json","@type":"CatalogPage","commitId":"69eb0337-b76d-44eb-921b-aa7e0efffcf8","commitTimeStamp":"2015-07-20T07:36:01.2104122Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page995.json","@type":"CatalogPage","commitId":"7d947f98-f9b9-4290-b201-87b6b7ed373e","commitTimeStamp":"2015-07-20T19:38:59.7131636Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page996.json","@type":"CatalogPage","commitId":"6b23fe99-6f89-4b5e-b34d-a9d89ad361a5","commitTimeStamp":"2015-07-21T08:19:01.000453Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page997.json","@type":"CatalogPage","commitId":"dbd669ae-e87a-440e-88fd-ab12fb1c0116","commitTimeStamp":"2015-07-21T18:10:38.530009Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page998.json","@type":"CatalogPage","commitId":"dbb9f5c0-0c88-4cc3-b7e2-f5b744c11dd5","commitTimeStamp":"2015-07-22T08:00:42.4409393Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page999.json","@type":"CatalogPage","commitId":"77d373a7-0953-4b8d-ab57-09aa5efb1dc2","commitTimeStamp":"2015-07-22T14:19:13.6817918Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page1000.json","@type":"CatalogPage","commitId":"38748a61-cc79-428a-a994-b924e92a9981","commitTimeStamp":"2015-07-23T00:14:59.5780714Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1001.json","@type":"CatalogPage","commitId":"7b4a812a-8ac5-4d53-8320-a6ff2abc14e4","commitTimeStamp":"2015-07-23T10:36:08.9166395Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1002.json","@type":"CatalogPage","commitId":"876b9d76-2316-4eae-bb9d-c683989347ea","commitTimeStamp":"2015-07-23T19:20:36.4266341Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1003.json","@type":"CatalogPage","commitId":"60d57116-8b83-4948-851b-e914dbc63b0e","commitTimeStamp":"2015-07-24T11:10:58.2059676Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1004.json","@type":"CatalogPage","commitId":"a493bf22-a96f-4cef-869e-2247317e27b7","commitTimeStamp":"2015-07-25T08:02:00.689977Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1005.json","@type":"CatalogPage","commitId":"adbb1b6b-ad68-493c-8bfb-6ea4484a6ed8","commitTimeStamp":"2015-07-26T18:16:22.5851192Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1006.json","@type":"CatalogPage","commitId":"7103b42d-6c21-4158-81e1-0923862368a3","commitTimeStamp":"2015-07-27T11:33:03.418905Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1007.json","@type":"CatalogPage","commitId":"4d324283-25b2-46b2-a435-b8a394fbd0ea","commitTimeStamp":"2015-07-27T18:29:29.8937737Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1008.json","@type":"CatalogPage","commitId":"a68a88e9-1c9f-4990-9197-010bf03f7bab","commitTimeStamp":"2015-07-28T09:31:04.5113901Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1009.json","@type":"CatalogPage","commitId":"e5676a8c-9fc4-4bd3-a023-a3d8afc7edeb","commitTimeStamp":"2015-07-28T21:14:17.506104Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1010.json","@type":"CatalogPage","commitId":"61b4ba7e-f812-4717-a79d-dd222a38705b","commitTimeStamp":"2015-07-29T09:15:07.9969673Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1011.json","@type":"CatalogPage","commitId":"435ecc6a-f5c5-4447-ac0f-53401c6dad58","commitTimeStamp":"2015-07-29T15:42:35.8773624Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1012.json","@type":"CatalogPage","commitId":"c4abfee2-7dca-4ba6-9e11-f68d4d475ece","commitTimeStamp":"2015-07-30T01:48:06.6465394Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page1013.json","@type":"CatalogPage","commitId":"df615807-cadc-441e-9e8f-5d94d9d2bbed","commitTimeStamp":"2015-07-30T12:27:15.614311Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1014.json","@type":"CatalogPage","commitId":"29a365e1-312e-4adf-884e-4baee9e21d36","commitTimeStamp":"2015-07-30T22:04:35.8303462Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1015.json","@type":"CatalogPage","commitId":"b16e1ac4-130b-4f5e-b6fe-15c7b58e96a1","commitTimeStamp":"2015-07-31T11:14:08.2922056Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1016.json","@type":"CatalogPage","commitId":"123d1125-965d-4aa0-a12f-d9ba70090fcb","commitTimeStamp":"2015-08-01T03:38:33.1155504Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1017.json","@type":"CatalogPage","commitId":"5d0883de-eafc-48ae-959a-1b28b4c00481","commitTimeStamp":"2015-08-02T06:05:47.9146096Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1018.json","@type":"CatalogPage","commitId":"bd7f5e90-c340-4590-b502-eeca4cc6ad25","commitTimeStamp":"2015-08-03T03:13:04.3859565Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1019.json","@type":"CatalogPage","commitId":"b72b1233-2d5f-40e0-b971-78a70e6b2da9","commitTimeStamp":"2015-08-03T19:25:35.2251119Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1020.json","@type":"CatalogPage","commitId":"09cab9b5-2238-4ddf-80a4-d1a7e419fc5d","commitTimeStamp":"2015-08-04T12:52:46.6202013Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1021.json","@type":"CatalogPage","commitId":"7919e056-f280-427d-8b10-bcef24150103","commitTimeStamp":"2015-08-05T00:24:01.8682053Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1022.json","@type":"CatalogPage","commitId":"9f19fcca-959a-4366-a8f4-f491a068f8ed","commitTimeStamp":"2015-08-05T19:33:47.3382602Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1023.json","@type":"CatalogPage","commitId":"0969f794-8184-48bc-8d8f-5ae2b3c3cf2e","commitTimeStamp":"2015-08-06T11:36:18.5278964Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1024.json","@type":"CatalogPage","commitId":"3280da5f-8e8d-449a-8e2d-b9e2609eea85","commitTimeStamp":"2015-08-06T20:08:44.197389Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page1025.json","@type":"CatalogPage","commitId":"d737ad48-3805-4c7e-8e20-5c3a96f627f5","commitTimeStamp":"2015-08-07T07:43:41.9618365Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1026.json","@type":"CatalogPage","commitId":"2b08ecd1-5e67-45ee-b63d-05663ff18213","commitTimeStamp":"2015-08-08T01:33:05.812028Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1027.json","@type":"CatalogPage","commitId":"8b2e974d-45b2-4198-ac8b-d48ff0b3a662","commitTimeStamp":"2015-08-08T23:47:09.1754849Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1028.json","@type":"CatalogPage","commitId":"7b2e59f7-f16c-44ad-9783-d5bbd5faf3d5","commitTimeStamp":"2015-08-10T03:20:02.8711102Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1029.json","@type":"CatalogPage","commitId":"fc19e638-a392-4f7b-81af-b10dedf8ccae","commitTimeStamp":"2015-08-10T19:57:21.0802628Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1030.json","@type":"CatalogPage","commitId":"f3fb2b6a-d1aa-4a42-b69e-7180c6d519b6","commitTimeStamp":"2015-08-11T14:09:36.8822476Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1031.json","@type":"CatalogPage","commitId":"a1b2ee3e-f673-4b31-b22a-b5ea56526de5","commitTimeStamp":"2015-08-12T03:02:27.3988701Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1032.json","@type":"CatalogPage","commitId":"af07a658-aadf-4a8b-8608-3d8658890240","commitTimeStamp":"2015-08-12T12:47:51.3615759Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1033.json","@type":"CatalogPage","commitId":"525932b9-b1ad-46e4-aada-7f8b9db5d467","commitTimeStamp":"2015-08-12T21:58:34.7188177Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1034.json","@type":"CatalogPage","commitId":"059c4a93-e53c-480a-8596-13d59810b8d5","commitTimeStamp":"2015-08-13T10:44:47.156238Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1035.json","@type":"CatalogPage","commitId":"a7c22932-f4de-42ad-ab4e-8d4406a4bd69","commitTimeStamp":"2015-08-13T18:18:42.4038206Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1036.json","@type":"CatalogPage","commitId":"7b8f794c-dcde-4603-b7d7-c2f2a04ee94e","commitTimeStamp":"2015-08-14T08:09:09.0906922Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1037.json","@type":"CatalogPage","commitId":"31c41cb3-97a6-41a4-b99e-c44bfdcfb195","commitTimeStamp":"2015-08-14T17:36:18.4728064Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1038.json","@type":"CatalogPage","commitId":"824562de-6c6d-45ca-b119-eaae5394343e","commitTimeStamp":"2015-08-15T06:58:01.1839258Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1039.json","@type":"CatalogPage","commitId":"f1133084-4a5e-40b2-a0f6-9abccc305d89","commitTimeStamp":"2015-08-16T13:04:29.3400053Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1040.json","@type":"CatalogPage","commitId":"632f45aa-99e2-4f2e-8358-e649e0637cd2","commitTimeStamp":"2015-08-17T12:35:44.0911277Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1041.json","@type":"CatalogPage","commitId":"882944d1-ccfd-47eb-a486-f7f9a2e241d2","commitTimeStamp":"2015-08-18T00:35:48.3322781Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1042.json","@type":"CatalogPage","commitId":"0e2188a3-180b-4d41-91f7-93ccfb948f1c","commitTimeStamp":"2015-08-18T17:54:41.2212562Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1043.json","@type":"CatalogPage","commitId":"ac772f7c-fcfd-4dd5-80c0-2396891ad8ce","commitTimeStamp":"2015-08-19T12:12:37.8660414Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1044.json","@type":"CatalogPage","commitId":"a7a5e1b9-9da2-4e55-baa0-854a844ad4da","commitTimeStamp":"2015-08-19T19:34:28.7498331Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1045.json","@type":"CatalogPage","commitId":"8a54714e-a767-4ae3-afc4-14bfcfccccf6","commitTimeStamp":"2015-08-20T11:58:23.7799642Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1046.json","@type":"CatalogPage","commitId":"e8933c8f-1c61-472d-92fb-35580c38f461","commitTimeStamp":"2015-08-21T04:46:00.2237858Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1047.json","@type":"CatalogPage","commitId":"33210397-ceb3-4d92-b41d-745b5bf67877","commitTimeStamp":"2015-08-21T18:56:11.3334585Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1048.json","@type":"CatalogPage","commitId":"b74a1e56-10c8-4e9a-a15e-56805cdd55cd","commitTimeStamp":"2015-08-22T23:17:02.011737Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1049.json","@type":"CatalogPage","commitId":"201c46b7-d355-47d0-a2c9-fa0d6710f477","commitTimeStamp":"2015-08-24T05:12:21.779258Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1050.json","@type":"CatalogPage","commitId":"c43d9247-1a91-4abb-89bc-52c780297ebb","commitTimeStamp":"2015-08-24T18:06:31.667194Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1051.json","@type":"CatalogPage","commitId":"e0eee1a0-9064-401b-93f3-0aa0d4f90aed","commitTimeStamp":"2015-08-25T10:39:41.8286193Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1052.json","@type":"CatalogPage","commitId":"7d757800-9dbe-4077-8ca6-28581376e138","commitTimeStamp":"2015-08-25T23:25:54.8671775Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1053.json","@type":"CatalogPage","commitId":"192318ae-72a3-457c-8293-101e90eaa805","commitTimeStamp":"2015-08-26T09:31:43.2772623Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1054.json","@type":"CatalogPage","commitId":"6e2b8314-67b5-4f26-a6a8-f11b75dcf889","commitTimeStamp":"2015-08-26T20:12:33.903492Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1055.json","@type":"CatalogPage","commitId":"6603dca0-5640-4aa4-8970-2449a736855c","commitTimeStamp":"2015-08-27T12:50:06.7938926Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1056.json","@type":"CatalogPage","commitId":"f4e8d192-35c9-4658-b926-d7cbb5347247","commitTimeStamp":"2015-08-28T00:21:41.1861275Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1057.json","@type":"CatalogPage","commitId":"95baaac3-64cc-4826-87ce-8f96ef577ec7","commitTimeStamp":"2015-08-28T16:16:02.8632073Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1058.json","@type":"CatalogPage","commitId":"7c9bee00-beeb-4300-ac0c-dabc82a7c258","commitTimeStamp":"2015-08-29T18:07:45.3985512Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1059.json","@type":"CatalogPage","commitId":"708ecf7a-46bf-4ee6-a232-0f01662dbff9","commitTimeStamp":"2015-08-30T08:36:38.0702302Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1060.json","@type":"CatalogPage","commitId":"2039b873-379a-4a48-9573-b0ee792fa980","commitTimeStamp":"2015-08-31T03:59:12.8394401Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1061.json","@type":"CatalogPage","commitId":"9327456a-5a60-4f27-9ff2-36593255ef55","commitTimeStamp":"2015-08-31T18:55:17.2066478Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1062.json","@type":"CatalogPage","commitId":"6466948d-852b-4f78-9e37-90edf52d03f5","commitTimeStamp":"2015-09-01T07:39:24.9995578Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page1063.json","@type":"CatalogPage","commitId":"bcb98247-f832-4849-94ef-b433d68fc7cb","commitTimeStamp":"2015-09-01T13:42:08.3484177Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1064.json","@type":"CatalogPage","commitId":"5ecf7fe0-fb49-4625-ac97-97d4fb80f1df","commitTimeStamp":"2015-09-01T20:15:30.187315Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1065.json","@type":"CatalogPage","commitId":"210c0ed0-ab31-4c3b-b0a5-41ec7e987685","commitTimeStamp":"2015-09-02T02:19:46.3238995Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1066.json","@type":"CatalogPage","commitId":"3c83efcc-3b87-471c-b54e-466593012915","commitTimeStamp":"2015-09-02T16:55:08.4569691Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1067.json","@type":"CatalogPage","commitId":"4e7dd2cc-0cf9-4d70-950d-581d9de84989","commitTimeStamp":"2015-09-03T07:09:55.1385033Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1068.json","@type":"CatalogPage","commitId":"e1500c88-0023-4074-8bef-60c8bb42b227","commitTimeStamp":"2015-09-03T17:43:01.4844087Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1069.json","@type":"CatalogPage","commitId":"5f57849a-3662-4a44-be24-b2287f9ec652","commitTimeStamp":"2015-09-04T10:12:10.4538249Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1070.json","@type":"CatalogPage","commitId":"770ce3fd-0d8b-4b2d-a486-7dab03eec44e","commitTimeStamp":"2015-09-04T22:56:14.8650969Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1071.json","@type":"CatalogPage","commitId":"2f842794-3d03-4556-89d7-8826e9fdccda","commitTimeStamp":"2015-09-06T02:24:05.6768546Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1072.json","@type":"CatalogPage","commitId":"14f3b40f-bba9-4466-9a3b-a8dc0d75dbc0","commitTimeStamp":"2015-09-06T21:16:09.0659432Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1073.json","@type":"CatalogPage","commitId":"9b7233de-809e-45ba-ab58-70e4192009a9","commitTimeStamp":"2015-09-07T15:25:04.8506275Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1074.json","@type":"CatalogPage","commitId":"07371744-2a67-41a0-aeec-97974b297528","commitTimeStamp":"2015-09-08T02:21:41.6740834Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1075.json","@type":"CatalogPage","commitId":"c6407b5f-142d-41b6-9845-5cab7df5fbbe","commitTimeStamp":"2015-09-08T14:05:53.5418214Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1076.json","@type":"CatalogPage","commitId":"a6b6e64d-53a2-47b1-8ce1-7084c741bb31","commitTimeStamp":"2015-09-09T04:10:00.4504001Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1077.json","@type":"CatalogPage","commitId":"d8eaf0a3-e728-4436-9123-6411d3d0f783","commitTimeStamp":"2015-09-09T17:07:26.8615975Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1078.json","@type":"CatalogPage","commitId":"98716f1f-466d-47fc-95ff-af9c1a6ad065","commitTimeStamp":"2015-09-10T09:34:39.0122902Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1079.json","@type":"CatalogPage","commitId":"89b40c7c-5902-4ad1-861a-6d6ef5b755b2","commitTimeStamp":"2015-09-10T21:46:04.9756589Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1080.json","@type":"CatalogPage","commitId":"bf5242f7-0cdb-4ae5-b8eb-4ee76978414d","commitTimeStamp":"2015-09-11T13:35:34.8179225Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1081.json","@type":"CatalogPage","commitId":"32d33ff7-f2c7-40a7-aace-19327ed0fdb5","commitTimeStamp":"2015-09-12T07:57:16.5091548Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1082.json","@type":"CatalogPage","commitId":"bfa64329-2275-4983-9de6-088fbecc9a83","commitTimeStamp":"2015-09-13T13:49:06.806229Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1083.json","@type":"CatalogPage","commitId":"3053a850-42e6-4929-ab90-bd09e2343bb9","commitTimeStamp":"2015-09-14T07:45:28.1641908Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1084.json","@type":"CatalogPage","commitId":"6de73b89-8a4f-4d0f-810b-df889dcb0884","commitTimeStamp":"2015-09-14T18:26:10.6100225Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1085.json","@type":"CatalogPage","commitId":"00ee3c6c-6a63-4db1-a00d-b271d987b789","commitTimeStamp":"2015-09-15T12:21:59.9168011Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1086.json","@type":"CatalogPage","commitId":"4beb9e51-dd35-4491-ac9e-b75e7c448e80","commitTimeStamp":"2015-09-16T07:01:29.6823709Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1087.json","@type":"CatalogPage","commitId":"536d40cf-93d3-444f-b2f0-f1dc26549a1f","commitTimeStamp":"2015-09-16T17:21:53.3679321Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1088.json","@type":"CatalogPage","commitId":"115104c1-ba12-43c5-ac60-2eb5907a55b8","commitTimeStamp":"2015-09-17T05:35:16.5515888Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1089.json","@type":"CatalogPage","commitId":"63b76ae5-a024-4b9b-bdde-30e65caa3439","commitTimeStamp":"2015-09-17T18:33:14.1644107Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1090.json","@type":"CatalogPage","commitId":"2da587b6-cef9-48dc-bf72-069eae2571cd","commitTimeStamp":"2015-09-18T11:31:33.7622927Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1091.json","@type":"CatalogPage","commitId":"d6eae5b7-6f86-4c8f-ad0f-731ab7f3cb5f","commitTimeStamp":"2015-09-18T23:42:05.8430418Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1092.json","@type":"CatalogPage","commitId":"379ee16f-921f-465c-9b3a-48e4aea8a5c1","commitTimeStamp":"2015-09-20T11:07:33.406439Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1093.json","@type":"CatalogPage","commitId":"eb755c6b-a794-4d44-a978-91de0bcaedc0","commitTimeStamp":"2015-09-21T11:03:52.5893849Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1094.json","@type":"CatalogPage","commitId":"76fa03c8-f81d-4480-bd69-38631d37d58c","commitTimeStamp":"2015-09-21T21:59:15.756433Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1095.json","@type":"CatalogPage","commitId":"58f5f855-ce6d-41c8-a12d-8fa5050565e2","commitTimeStamp":"2015-09-22T11:39:55.211341Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1096.json","@type":"CatalogPage","commitId":"be704f17-1b54-4f36-8cf3-6d6e314459fe","commitTimeStamp":"2015-09-22T20:27:24.799278Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1097.json","@type":"CatalogPage","commitId":"35dde721-d805-4d8f-9051-0efb91cf5d36","commitTimeStamp":"2015-09-23T09:38:10.7650885Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1098.json","@type":"CatalogPage","commitId":"2d2e573d-c02e-454a-b7df-2b4ee605549f","commitTimeStamp":"2015-09-23T23:19:31.2138508Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1099.json","@type":"CatalogPage","commitId":"c6743e1d-0b99-4fa5-ae82-d46526c66f4c","commitTimeStamp":"2015-09-24T12:43:24.6513763Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1100.json","@type":"CatalogPage","commitId":"acdf94f0-b339-48f8-bd76-ab945f19d30c","commitTimeStamp":"2015-09-25T07:45:33.8244123Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1101.json","@type":"CatalogPage","commitId":"9cbbd92b-6e03-4c8d-a40a-5af1286c8ace","commitTimeStamp":"2015-09-25T18:41:40.267258Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1102.json","@type":"CatalogPage","commitId":"6b6abc64-7d2c-463c-a333-8c6010234981","commitTimeStamp":"2015-09-26T18:26:17.6307813Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1103.json","@type":"CatalogPage","commitId":"88cd272c-fe7e-41ba-bdca-e73a938ab90f","commitTimeStamp":"2015-09-27T22:51:21.2482324Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1104.json","@type":"CatalogPage","commitId":"72fe5431-8b00-485f-bc1a-26871377177a","commitTimeStamp":"2015-09-28T12:55:06.2879569Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1105.json","@type":"CatalogPage","commitId":"1e5680f6-28fe-4b99-a524-2ae28045a5b5","commitTimeStamp":"2015-09-29T06:31:07.4252301Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1106.json","@type":"CatalogPage","commitId":"249d60c5-e562-4fb9-a26c-2938b1b45321","commitTimeStamp":"2015-09-29T20:26:51.0668927Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1107.json","@type":"CatalogPage","commitId":"d05d50c0-04a1-4506-a0be-c72722a191ad","commitTimeStamp":"2015-09-30T14:40:58.0462581Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1108.json","@type":"CatalogPage","commitId":"9a4d30d6-6bf7-4530-ba03-1d640ed53aac","commitTimeStamp":"2015-09-30T21:15:14.471136Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1109.json","@type":"CatalogPage","commitId":"7a97ecee-8c9d-431d-b7d2-b520e8cee410","commitTimeStamp":"2015-10-01T12:49:57.0306955Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1110.json","@type":"CatalogPage","commitId":"1efbac15-79d2-489b-86f6-4d8083321d1f","commitTimeStamp":"2015-10-02T00:17:29.3638859Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1111.json","@type":"CatalogPage","commitId":"91ce9c89-47b1-43c2-8884-50d46d7fda95","commitTimeStamp":"2015-10-02T14:09:33.4726319Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1112.json","@type":"CatalogPage","commitId":"4b68eeb9-186c-4012-ba04-b92cb3b7cb13","commitTimeStamp":"2015-10-03T13:24:02.1605243Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1113.json","@type":"CatalogPage","commitId":"b1cd11c2-4716-436a-8919-843f754a5a9e","commitTimeStamp":"2015-10-04T04:28:58.2152123Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1114.json","@type":"CatalogPage","commitId":"9d91f212-40c6-498c-91ed-847fe6d5d522","commitTimeStamp":"2015-10-04T07:41:52.7804757Z","count":533},{"@id":"https://api.nuget.org/v3/catalog0/page1115.json","@type":"CatalogPage","commitId":"92f6ed32-b6ec-4a1d-9faa-42c9d27f4135","commitTimeStamp":"2015-10-04T07:45:51.1230979Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1116.json","@type":"CatalogPage","commitId":"c4e8c2e4-9075-44dc-9468-7f53b802839b","commitTimeStamp":"2015-10-04T07:48:58.8658581Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1117.json","@type":"CatalogPage","commitId":"8fe02e74-aa3b-498c-8f40-438d0f22e79c","commitTimeStamp":"2015-10-04T07:51:41.6226398Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1118.json","@type":"CatalogPage","commitId":"1d437cb4-7e37-44f4-b5c7-6df53a9fa284","commitTimeStamp":"2015-10-04T07:54:49.2215187Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1119.json","@type":"CatalogPage","commitId":"4b39b1ce-bc90-4bee-be82-2541c1cd3fbe","commitTimeStamp":"2015-10-04T07:57:49.3221129Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1120.json","@type":"CatalogPage","commitId":"418b7b51-4b52-489b-a51d-5494bcb5c89d","commitTimeStamp":"2015-10-04T08:01:24.2299994Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1121.json","@type":"CatalogPage","commitId":"fcc5b1e9-cd54-480f-afc6-d65b043ab552","commitTimeStamp":"2015-10-04T08:05:14.7498338Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1122.json","@type":"CatalogPage","commitId":"d4ab37e1-b906-4d5f-a83c-8698ffd9b51d","commitTimeStamp":"2015-10-05T00:34:30.2455219Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1123.json","@type":"CatalogPage","commitId":"4cd30ad4-c690-4a61-9afd-77c4909e701a","commitTimeStamp":"2015-10-05T15:37:49.7190373Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1124.json","@type":"CatalogPage","commitId":"ad46fd1b-38f4-4f70-b3b6-c91c7b8c57b1","commitTimeStamp":"2015-10-06T09:45:26.0884845Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1125.json","@type":"CatalogPage","commitId":"a60be053-cbd4-45f2-83a3-cd111359353d","commitTimeStamp":"2015-10-06T18:02:03.908456Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1126.json","@type":"CatalogPage","commitId":"a1c3b14f-16f9-4ba4-b6dc-9526d0396988","commitTimeStamp":"2015-10-07T08:18:24.2564183Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1127.json","@type":"CatalogPage","commitId":"e2b00f90-5cfb-4408-8c8e-a1a125d04e97","commitTimeStamp":"2015-10-07T17:19:31.2480528Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1128.json","@type":"CatalogPage","commitId":"9fbf18de-26d2-4e03-977f-ee9f2717cb16","commitTimeStamp":"2015-10-08T10:12:14.9674096Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1129.json","@type":"CatalogPage","commitId":"ecc732c7-1be6-4bc9-b2c3-9db9c75bc847","commitTimeStamp":"2015-10-08T18:01:34.7471491Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1130.json","@type":"CatalogPage","commitId":"71f9a227-5615-4e4c-949b-d7a929bbb986","commitTimeStamp":"2015-10-09T10:48:21.7893432Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1131.json","@type":"CatalogPage","commitId":"0454b4de-536d-4634-b6c1-2a0d5779416b","commitTimeStamp":"2015-10-10T08:09:01.5049984Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1132.json","@type":"CatalogPage","commitId":"4ef0169b-9248-41a5-89cc-f66d7208e399","commitTimeStamp":"2015-10-11T03:50:29.0587324Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1133.json","@type":"CatalogPage","commitId":"689264f7-602e-4662-bf28-7f92e6099226","commitTimeStamp":"2015-10-12T09:09:38.3667946Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1134.json","@type":"CatalogPage","commitId":"26702433-0b23-4e7c-9d0b-d0f117d4c867","commitTimeStamp":"2015-10-12T19:56:28.0884643Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1135.json","@type":"CatalogPage","commitId":"9cbb4106-3805-4fad-9dfa-e8d9f6aac100","commitTimeStamp":"2015-10-13T12:38:24.9435514Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1136.json","@type":"CatalogPage","commitId":"095dcee7-0a42-4aa5-a685-b444200333bf","commitTimeStamp":"2015-10-13T21:58:51.5856953Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1137.json","@type":"CatalogPage","commitId":"a37f5cc9-cc70-4e76-a109-b37c7a17fd10","commitTimeStamp":"2015-10-14T09:18:56.6951747Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1138.json","@type":"CatalogPage","commitId":"c5f9a10b-302b-473f-94e9-a35bd54ad7b8","commitTimeStamp":"2015-10-14T19:33:40.7568507Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1139.json","@type":"CatalogPage","commitId":"30006da1-6a83-4dee-aaee-73e53a9f70bd","commitTimeStamp":"2015-10-15T09:16:16.8448528Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1140.json","@type":"CatalogPage","commitId":"e77e0734-ff96-412c-a959-cfe25e74dccf","commitTimeStamp":"2015-10-15T15:53:45.4680135Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1141.json","@type":"CatalogPage","commitId":"fe4d9e3b-2c8b-4449-8229-ac95301f3668","commitTimeStamp":"2015-10-15T19:27:22.4420776Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1142.json","@type":"CatalogPage","commitId":"cd5055cc-fd5f-4ed9-b77c-506e52dde05d","commitTimeStamp":"2015-10-16T09:39:06.7981448Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1143.json","@type":"CatalogPage","commitId":"bc8b7b0a-d8ed-4a16-ac97-af66629d69be","commitTimeStamp":"2015-10-16T16:02:41.8230693Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1144.json","@type":"CatalogPage","commitId":"7a27976d-e77a-40f3-902b-c58cee577293","commitTimeStamp":"2015-10-17T14:35:11.9220389Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1145.json","@type":"CatalogPage","commitId":"93bd6944-b42e-4be6-9718-4e6183ac6747","commitTimeStamp":"2015-10-18T20:36:45.9552377Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1146.json","@type":"CatalogPage","commitId":"1a034783-b73f-4d88-a953-5a25816a6423","commitTimeStamp":"2015-10-19T12:20:25.6575055Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1147.json","@type":"CatalogPage","commitId":"55b7d251-ac69-483d-9f1a-df3019fa9c80","commitTimeStamp":"2015-10-19T22:37:36.4395168Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1148.json","@type":"CatalogPage","commitId":"43d7b2b0-a7f2-4a85-ae4d-af864872a60e","commitTimeStamp":"2015-10-20T11:57:35.5450752Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1149.json","@type":"CatalogPage","commitId":"9ea69064-6a3e-4c1f-b588-176b6b0f3f0d","commitTimeStamp":"2015-10-20T22:47:18.6344016Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1150.json","@type":"CatalogPage","commitId":"da518f48-f995-4251-bca6-376c203f83d3","commitTimeStamp":"2015-10-21T15:40:23.0597507Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1151.json","@type":"CatalogPage","commitId":"2df284e1-0185-4a1e-b1b1-a9a476588974","commitTimeStamp":"2015-10-22T07:12:21.9782247Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1152.json","@type":"CatalogPage","commitId":"fd1fd37f-7176-4e37-8d0d-a8b38a2bf0cd","commitTimeStamp":"2015-10-22T19:21:34.2727966Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1153.json","@type":"CatalogPage","commitId":"768110f4-ee45-40be-a55d-8748082902f0","commitTimeStamp":"2015-10-23T12:21:03.022191Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1154.json","@type":"CatalogPage","commitId":"446af7a4-b74d-4919-95a4-9227b695fd21","commitTimeStamp":"2015-10-24T09:56:49.1692989Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1155.json","@type":"CatalogPage","commitId":"ac4d0cdd-8982-4850-9af3-3e621a396f0f","commitTimeStamp":"2015-10-25T03:52:37.5949348Z","count":533},{"@id":"https://api.nuget.org/v3/catalog0/page1156.json","@type":"CatalogPage","commitId":"82526609-6961-4784-a1ff-eb173839f183","commitTimeStamp":"2015-10-26T02:37:34.2641085Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1157.json","@type":"CatalogPage","commitId":"7b188bb4-3de2-496a-8a29-e1802c955533","commitTimeStamp":"2015-10-26T15:51:04.8440061Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1158.json","@type":"CatalogPage","commitId":"14381849-9612-41e7-adfb-ad627d905cff","commitTimeStamp":"2015-10-27T04:58:52.9427071Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1159.json","@type":"CatalogPage","commitId":"0a9db8b2-3388-47a7-a566-c595fc100df9","commitTimeStamp":"2015-10-27T14:48:43.9955105Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page1160.json","@type":"CatalogPage","commitId":"75ba5489-6a21-4f91-9184-9ab9c44fb0a2","commitTimeStamp":"2015-10-27T17:09:25.9664365Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1161.json","@type":"CatalogPage","commitId":"d98da0de-b6fc-482a-b829-6c0b771574c2","commitTimeStamp":"2015-10-28T09:17:56.2441213Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1162.json","@type":"CatalogPage","commitId":"a5adfc67-66de-4502-9a56-721494967bd3","commitTimeStamp":"2015-10-28T17:58:43.6632928Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1163.json","@type":"CatalogPage","commitId":"56c4a703-c718-410b-8b16-998c75857be8","commitTimeStamp":"2015-10-29T09:25:08.7257005Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1164.json","@type":"CatalogPage","commitId":"ac764ec1-4c7a-44b7-80d7-94f317e4ecf4","commitTimeStamp":"2015-10-29T17:41:55.036968Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1165.json","@type":"CatalogPage","commitId":"2c130d4d-382e-4882-a694-dd41c527b942","commitTimeStamp":"2015-10-30T09:25:25.9829614Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1166.json","@type":"CatalogPage","commitId":"c9b8cace-29a8-4d9a-8fb9-e447054bdae8","commitTimeStamp":"2015-10-31T01:17:29.6714107Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1167.json","@type":"CatalogPage","commitId":"c44dcf2f-0149-43fa-850b-dcb110fb2797","commitTimeStamp":"2015-11-01T04:35:05.5129847Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1168.json","@type":"CatalogPage","commitId":"4c022714-2a75-4756-91cc-601ec4913b28","commitTimeStamp":"2015-11-01T23:44:47.1498726Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1169.json","@type":"CatalogPage","commitId":"914cd4a5-6e97-4b2d-9efd-eb376832c00c","commitTimeStamp":"2015-11-02T11:43:25.2113988Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1170.json","@type":"CatalogPage","commitId":"eabdb469-e14c-4b01-86a8-5307c9ddef7b","commitTimeStamp":"2015-11-02T21:00:57.460113Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1171.json","@type":"CatalogPage","commitId":"e2193186-0857-4ebe-a604-407c8cdf7815","commitTimeStamp":"2015-11-03T13:32:17.4473024Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1172.json","@type":"CatalogPage","commitId":"77956395-b17c-4fdd-af54-0eb1303f3bf7","commitTimeStamp":"2015-11-04T00:40:13.0589773Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1173.json","@type":"CatalogPage","commitId":"723ca571-ec44-4e00-beb6-e382c823ed3c","commitTimeStamp":"2015-11-04T15:38:15.5493394Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1174.json","@type":"CatalogPage","commitId":"deb60173-bc94-42d7-b244-a0fa7d1494e7","commitTimeStamp":"2015-11-05T05:38:42.1678138Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1175.json","@type":"CatalogPage","commitId":"3e30256f-0421-400b-be4b-642c1cd37766","commitTimeStamp":"2015-11-05T19:43:15.2134458Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1176.json","@type":"CatalogPage","commitId":"d46e49d4-ebd5-4e6b-a61e-06428d353026","commitTimeStamp":"2015-11-06T10:18:44.2272196Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1177.json","@type":"CatalogPage","commitId":"7a65ce36-f7ae-4873-ab58-62882c07acdd","commitTimeStamp":"2015-11-06T21:43:42.9249146Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1178.json","@type":"CatalogPage","commitId":"10c21bbf-10cc-485c-bf20-c8c4c0aba2da","commitTimeStamp":"2015-11-08T07:46:02.6762837Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1179.json","@type":"CatalogPage","commitId":"35cc664a-b161-4eff-9d98-54e9438b1bd0","commitTimeStamp":"2015-11-09T05:46:53.3037863Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1180.json","@type":"CatalogPage","commitId":"ce72850c-c9e5-4480-9933-8ed496e79341","commitTimeStamp":"2015-11-09T19:15:42.7160333Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1181.json","@type":"CatalogPage","commitId":"36e0bc8c-263b-4cb8-9af4-672bc27a8e91","commitTimeStamp":"2015-11-10T14:07:04.1858693Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1182.json","@type":"CatalogPage","commitId":"fe867b96-69f6-4590-be45-3f0cbf1c2a15","commitTimeStamp":"2015-11-11T02:55:02.5639286Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1183.json","@type":"CatalogPage","commitId":"9c4a0f67-d94e-4b25-a05a-637929b7663b","commitTimeStamp":"2015-11-11T18:01:30.8460633Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1184.json","@type":"CatalogPage","commitId":"6e1bf44a-03d7-436e-9d45-de24e9205fd2","commitTimeStamp":"2015-11-12T08:03:52.0861041Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1185.json","@type":"CatalogPage","commitId":"135e1bc5-94c3-4563-81af-0c551fc73a52","commitTimeStamp":"2015-11-12T16:11:43.1505429Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1186.json","@type":"CatalogPage","commitId":"52676fc9-e0d5-428d-bed1-42ddc155cd37","commitTimeStamp":"2015-11-13T06:56:51.4418874Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1187.json","@type":"CatalogPage","commitId":"e9566ee9-89f9-4c5a-8292-3e3078f87582","commitTimeStamp":"2015-11-13T17:15:08.5309646Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1188.json","@type":"CatalogPage","commitId":"8348944f-9485-4521-a517-72759573b0ee","commitTimeStamp":"2015-11-14T13:46:16.011612Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1189.json","@type":"CatalogPage","commitId":"ebeedaad-dab3-4937-b759-39fef400233e","commitTimeStamp":"2015-11-15T15:24:45.1987126Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1190.json","@type":"CatalogPage","commitId":"b4adbbf9-aab7-4888-abd1-b23446ff69c9","commitTimeStamp":"2015-11-16T11:07:32.0276633Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1191.json","@type":"CatalogPage","commitId":"6bc9070b-5712-46bb-8d9f-f3b37aa28edf","commitTimeStamp":"2015-11-16T19:32:01.1697344Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1192.json","@type":"CatalogPage","commitId":"ca351e86-50f9-4835-9ee7-593276b49fda","commitTimeStamp":"2015-11-17T12:19:00.4923516Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1193.json","@type":"CatalogPage","commitId":"0fb78389-da77-4ea9-a939-8140b2047e92","commitTimeStamp":"2015-11-17T20:10:15.3531397Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1194.json","@type":"CatalogPage","commitId":"33b75393-72b4-4c3c-a299-31e75e95ae7a","commitTimeStamp":"2015-11-18T12:55:36.6382892Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1195.json","@type":"CatalogPage","commitId":"79636d70-14fa-4f07-8ffa-09ff045596f5","commitTimeStamp":"2015-11-18T16:32:54.6512386Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1196.json","@type":"CatalogPage","commitId":"06b2020c-5756-4a8b-81df-9ad946a15fa0","commitTimeStamp":"2015-11-19T09:18:30.042033Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1197.json","@type":"CatalogPage","commitId":"74529395-a50a-4cba-8e0c-34a100402a66","commitTimeStamp":"2015-11-19T18:58:10.5651505Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1198.json","@type":"CatalogPage","commitId":"b6fea39c-ed0a-4c76-ac3b-a29fe23bc69d","commitTimeStamp":"2015-11-20T10:34:30.0321991Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1199.json","@type":"CatalogPage","commitId":"3df47a76-0e47-4b73-ac7f-e5be658f2686","commitTimeStamp":"2015-11-20T19:14:25.2815836Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1200.json","@type":"CatalogPage","commitId":"c64e1c95-b50a-4fa3-bfc6-c061401ef3f0","commitTimeStamp":"2015-11-21T15:48:39.2648508Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1201.json","@type":"CatalogPage","commitId":"fcd5a542-6d88-45fb-9f8e-85c491051d49","commitTimeStamp":"2015-11-23T00:58:59.0393366Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1202.json","@type":"CatalogPage","commitId":"2fc130cc-bdd7-4352-98a3-082b401f6c73","commitTimeStamp":"2015-11-23T14:50:32.3256484Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1203.json","@type":"CatalogPage","commitId":"d0fb64c3-3c32-413e-be5d-766be056ee8e","commitTimeStamp":"2015-11-24T08:29:07.5489969Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1204.json","@type":"CatalogPage","commitId":"2ecd200a-8d0f-4773-8074-622c14ee8fed","commitTimeStamp":"2015-11-24T16:55:11.4579157Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1205.json","@type":"CatalogPage","commitId":"c910e88e-b0b5-477c-85c8-18207a1968da","commitTimeStamp":"2015-11-25T08:09:28.5378342Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1206.json","@type":"CatalogPage","commitId":"5552df73-125d-4363-afb2-e0771c80c988","commitTimeStamp":"2015-11-25T18:17:10.4759678Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1207.json","@type":"CatalogPage","commitId":"00a49830-769e-4684-bf16-5a32b12b75e3","commitTimeStamp":"2015-11-26T08:33:51.9676544Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1208.json","@type":"CatalogPage","commitId":"99ce8e84-c2b3-4ac0-b912-02cb7f71d043","commitTimeStamp":"2015-11-26T18:54:10.0119672Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1209.json","@type":"CatalogPage","commitId":"e4b83f5d-fccd-4af0-bd63-8a47a3ad5974","commitTimeStamp":"2015-11-27T12:47:35.0013812Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1210.json","@type":"CatalogPage","commitId":"6fc4f9c4-fb8e-4cd9-9b1d-2b9590713c69","commitTimeStamp":"2015-11-28T02:27:24.7316448Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1211.json","@type":"CatalogPage","commitId":"9b3252e0-0d29-4ea9-9ca0-6fc85f1caaa0","commitTimeStamp":"2015-11-28T19:16:29.6147853Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1212.json","@type":"CatalogPage","commitId":"8c3025de-1601-4387-9d22-f7619d4fe5a2","commitTimeStamp":"2015-11-29T23:08:07.2500752Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1213.json","@type":"CatalogPage","commitId":"208abca5-a14f-4ae9-9dff-2e6d5867f258","commitTimeStamp":"2015-11-30T13:35:30.1848561Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1214.json","@type":"CatalogPage","commitId":"083295a7-0ce5-4e99-9e68-d80f8c1ea935","commitTimeStamp":"2015-12-01T04:01:26.8824971Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1215.json","@type":"CatalogPage","commitId":"4539e865-e3dc-42e3-93af-6235f5da57c4","commitTimeStamp":"2015-12-01T14:09:36.3549133Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1216.json","@type":"CatalogPage","commitId":"25292f72-7350-42e3-a9e8-cc33fc45c68d","commitTimeStamp":"2015-12-01T20:34:24.9428498Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1217.json","@type":"CatalogPage","commitId":"2f20259c-e8e6-4f90-9fde-c1e42be18561","commitTimeStamp":"2015-12-02T08:00:49.3608094Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1218.json","@type":"CatalogPage","commitId":"acb76f8d-fe5b-4fe3-b419-ce504dd803e4","commitTimeStamp":"2015-12-02T14:57:48.3207646Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page1219.json","@type":"CatalogPage","commitId":"36400169-41df-44b8-aef5-8fd665560e2a","commitTimeStamp":"2015-12-03T04:11:38.2964827Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1220.json","@type":"CatalogPage","commitId":"430cb892-f7bb-48dd-a879-c52bbaddef59","commitTimeStamp":"2015-12-03T14:42:57.4559666Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1221.json","@type":"CatalogPage","commitId":"4fc1b04d-7da8-4a5b-a9eb-e410d9e7e059","commitTimeStamp":"2015-12-03T22:09:58.7590949Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1222.json","@type":"CatalogPage","commitId":"2ec69b52-5ba3-4c7b-af5a-9af29b0d43ab","commitTimeStamp":"2015-12-04T13:45:09.5723183Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1223.json","@type":"CatalogPage","commitId":"b48932e5-7280-435e-88e2-d591585d266f","commitTimeStamp":"2015-12-05T06:31:29.5956857Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1224.json","@type":"CatalogPage","commitId":"461f4e73-abd5-4406-882e-750978c5f465","commitTimeStamp":"2015-12-06T10:42:16.0622672Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1225.json","@type":"CatalogPage","commitId":"30fc7a87-6cec-4ace-bd7f-6ef144324c53","commitTimeStamp":"2015-12-07T09:18:20.1657619Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1226.json","@type":"CatalogPage","commitId":"cca6124d-39d7-42ff-8612-b01f2b0e3c51","commitTimeStamp":"2015-12-07T20:59:35.0002203Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1227.json","@type":"CatalogPage","commitId":"03de1ece-9bb9-4f92-9066-b0ce9e8a2471","commitTimeStamp":"2015-12-08T09:53:29.4488172Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1228.json","@type":"CatalogPage","commitId":"c573fae7-9d68-4270-8e07-daffa45418a9","commitTimeStamp":"2015-12-08T18:23:41.5580672Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1229.json","@type":"CatalogPage","commitId":"e785fb02-c46c-45aa-8b84-72d79125883f","commitTimeStamp":"2015-12-09T10:43:02.0085642Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1230.json","@type":"CatalogPage","commitId":"c7fb6c05-67e9-440a-841e-907a77712471","commitTimeStamp":"2015-12-09T18:58:54.8814747Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1231.json","@type":"CatalogPage","commitId":"a96c9949-87b3-4e13-b035-7984086b107d","commitTimeStamp":"2015-12-10T05:39:41.0599789Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page1232.json","@type":"CatalogPage","commitId":"fd84c014-1c34-424b-8447-5535de60a247","commitTimeStamp":"2015-12-10T14:13:40.9897139Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1233.json","@type":"CatalogPage","commitId":"06d776ab-2110-483e-b98f-e05e03334262","commitTimeStamp":"2015-12-10T19:51:45.1973671Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page1234.json","@type":"CatalogPage","commitId":"84c44e1d-31b7-4231-9cce-256fc47b51a3","commitTimeStamp":"2015-12-11T05:57:05.5399775Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page1235.json","@type":"CatalogPage","commitId":"246809c3-13d0-42a7-b3de-772d0effce8c","commitTimeStamp":"2015-12-11T11:32:55.6484105Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1236.json","@type":"CatalogPage","commitId":"083e0fc7-4077-4c4e-97e9-98a32f98e3b5","commitTimeStamp":"2015-12-11T16:34:52.0239177Z","count":554},{"@id":"https://api.nuget.org/v3/catalog0/page1237.json","@type":"CatalogPage","commitId":"48824139-dd7a-4918-b011-44d81c283d0a","commitTimeStamp":"2015-12-12T03:25:47.4265328Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1238.json","@type":"CatalogPage","commitId":"cee8c4d0-49e0-408f-b313-93ec1bcdf2fc","commitTimeStamp":"2015-12-12T19:01:23.2766958Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1239.json","@type":"CatalogPage","commitId":"040e645f-7271-4d63-b4f7-f9bcdce65aed","commitTimeStamp":"2015-12-13T17:06:54.4057517Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1240.json","@type":"CatalogPage","commitId":"e377f654-120b-4eff-ae3e-07a108f91e9d","commitTimeStamp":"2015-12-14T09:47:20.2913091Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1241.json","@type":"CatalogPage","commitId":"962367b3-bb54-425d-8144-62c7a872df34","commitTimeStamp":"2015-12-14T15:19:55.3652265Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1242.json","@type":"CatalogPage","commitId":"620cfdbe-cb0e-450c-b540-c7e32392d0bf","commitTimeStamp":"2015-12-15T06:09:06.3648576Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1243.json","@type":"CatalogPage","commitId":"ada4c796-519f-48f7-b9fb-4459086b68dc","commitTimeStamp":"2015-12-15T17:09:34.676873Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1244.json","@type":"CatalogPage","commitId":"202fa187-29e6-4ada-8b8e-76382e982e15","commitTimeStamp":"2015-12-15T23:55:50.0075095Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page1245.json","@type":"CatalogPage","commitId":"9d9db4f2-209e-4c90-b5e5-7088ca8e9e5c","commitTimeStamp":"2015-12-16T14:03:55.8433985Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1246.json","@type":"CatalogPage","commitId":"6bbebe11-63f5-4e31-9783-b17d93ab8659","commitTimeStamp":"2015-12-17T03:57:23.6005067Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1247.json","@type":"CatalogPage","commitId":"9dfc7fc9-824d-42b0-9f46-40080ea1e791","commitTimeStamp":"2015-12-17T16:35:05.0577391Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1248.json","@type":"CatalogPage","commitId":"fb41cc04-496b-4520-b329-d19473319324","commitTimeStamp":"2015-12-18T07:16:10.229815Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1249.json","@type":"CatalogPage","commitId":"7751f2c3-75ec-4d91-b226-61b61976415a","commitTimeStamp":"2015-12-18T17:49:36.1347509Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1250.json","@type":"CatalogPage","commitId":"e1f2f27f-b09d-459d-97e0-b37a28b3e9b5","commitTimeStamp":"2015-12-19T15:00:23.7287684Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1251.json","@type":"CatalogPage","commitId":"f01a04d0-c1e4-40cc-a0ba-5319b8c20c17","commitTimeStamp":"2015-12-20T05:19:44.3136067Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1252.json","@type":"CatalogPage","commitId":"ecb42bbf-e099-4daa-956f-af549490c060","commitTimeStamp":"2015-12-21T13:08:09.6553121Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1253.json","@type":"CatalogPage","commitId":"4edd7303-ef17-4a22-bdc4-bc79dd569518","commitTimeStamp":"2015-12-22T06:36:15.6061356Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1254.json","@type":"CatalogPage","commitId":"90621b39-881d-4c7e-a3b1-e405f8bc48ec","commitTimeStamp":"2015-12-22T17:05:31.8013188Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1255.json","@type":"CatalogPage","commitId":"28c2e806-7702-475a-bb59-a38c760a6a63","commitTimeStamp":"2015-12-23T04:07:03.095502Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1256.json","@type":"CatalogPage","commitId":"9b4e5030-8ecb-4cb2-8f02-f4b3e519b881","commitTimeStamp":"2015-12-23T09:23:22.7595487Z","count":536},{"@id":"https://api.nuget.org/v3/catalog0/page1257.json","@type":"CatalogPage","commitId":"67e2b87d-57e8-4c09-91ac-3f78af5a9e21","commitTimeStamp":"2015-12-23T15:56:50.1605415Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1258.json","@type":"CatalogPage","commitId":"737ce7b6-bcb9-42b5-8e7e-a85b94c81d5b","commitTimeStamp":"2015-12-24T09:46:12.3177653Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page1259.json","@type":"CatalogPage","commitId":"0f67a53e-59ac-4690-b2e8-8c5853636ab9","commitTimeStamp":"2015-12-24T15:41:22.4539619Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1260.json","@type":"CatalogPage","commitId":"4827a0bd-130a-49c5-b114-8dc9608b5406","commitTimeStamp":"2015-12-25T08:14:25.0427142Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1261.json","@type":"CatalogPage","commitId":"d71a83aa-096a-4cb2-8570-a480fc26532d","commitTimeStamp":"2015-12-26T02:43:43.3699155Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1262.json","@type":"CatalogPage","commitId":"59271934-6ab5-48dd-a3b6-dffb4697ed2f","commitTimeStamp":"2015-12-27T02:48:27.634649Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1263.json","@type":"CatalogPage","commitId":"3402b14c-2614-4be7-bbc9-ac110a1db263","commitTimeStamp":"2015-12-27T22:38:48.5240233Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1264.json","@type":"CatalogPage","commitId":"0ea13e5b-20bd-4bc2-9769-e4576a200b51","commitTimeStamp":"2015-12-28T17:46:09.4357295Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1265.json","@type":"CatalogPage","commitId":"2eaf6981-3c99-4d1c-9ef7-f0293b2d9cee","commitTimeStamp":"2015-12-29T09:41:48.2103068Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1266.json","@type":"CatalogPage","commitId":"7bbf5d37-8b91-4197-90f2-2b91e21bb180","commitTimeStamp":"2015-12-29T20:47:05.1305069Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1267.json","@type":"CatalogPage","commitId":"4b1c8361-6cf7-4b97-b3a8-572d8dd8bce4","commitTimeStamp":"2015-12-30T16:40:59.5728392Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1268.json","@type":"CatalogPage","commitId":"3b865561-72b7-441f-9bac-a8f25a0f942e","commitTimeStamp":"2015-12-31T15:00:18.9624292Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1269.json","@type":"CatalogPage","commitId":"28d66861-3c0f-4def-aaeb-b1fca71d6002","commitTimeStamp":"2016-01-01T21:13:12.1905966Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1270.json","@type":"CatalogPage","commitId":"f4ff6e17-b722-4b63-8852-66b4b6516f49","commitTimeStamp":"2016-01-03T01:47:55.1056238Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1271.json","@type":"CatalogPage","commitId":"fc06d0cc-875a-469c-b10a-518c4e89a7ea","commitTimeStamp":"2016-01-04T01:05:26.0974103Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1272.json","@type":"CatalogPage","commitId":"6cb54ede-9f29-49c9-9af1-9af5bbbf7529","commitTimeStamp":"2016-01-04T16:10:12.074023Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1273.json","@type":"CatalogPage","commitId":"2dee47ad-ef0f-4807-ad85-a8743f160f54","commitTimeStamp":"2016-01-05T08:42:19.0999329Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1274.json","@type":"CatalogPage","commitId":"e054f262-d86a-4951-a371-1e7662c8f479","commitTimeStamp":"2016-01-05T16:01:11.8880426Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1275.json","@type":"CatalogPage","commitId":"618a3cc8-7161-48a5-9447-d89e6cfc84e8","commitTimeStamp":"2016-01-06T04:09:37.1279396Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1276.json","@type":"CatalogPage","commitId":"672367f1-95d3-4ab3-a538-e63768051d9e","commitTimeStamp":"2016-01-06T13:41:48.7864889Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1277.json","@type":"CatalogPage","commitId":"47be32df-6c0d-4e22-aacf-78781eafde86","commitTimeStamp":"2016-01-06T20:37:29.9372472Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1278.json","@type":"CatalogPage","commitId":"d3f5189b-f0cb-4b0d-9eb8-7c2372982fea","commitTimeStamp":"2016-01-07T06:50:59.8827912Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1279.json","@type":"CatalogPage","commitId":"ca6194a1-e71e-428d-b3c3-0fd70707d94a","commitTimeStamp":"2016-01-07T16:04:19.6824671Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1280.json","@type":"CatalogPage","commitId":"6d2defca-66bf-4f49-80a6-42169fe8a325","commitTimeStamp":"2016-01-08T02:04:31.9572146Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1281.json","@type":"CatalogPage","commitId":"c07788bd-69c3-4613-80eb-c52232afc1c9","commitTimeStamp":"2016-01-08T13:31:44.6111172Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1282.json","@type":"CatalogPage","commitId":"44fc3a87-bc88-45a1-bdec-cb30317ded15","commitTimeStamp":"2016-01-08T23:53:21.2585494Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1283.json","@type":"CatalogPage","commitId":"fc574007-390a-471a-bb64-d1c6c1fa2092","commitTimeStamp":"2016-01-09T12:17:27.4738887Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1284.json","@type":"CatalogPage","commitId":"c35f01b5-5e50-49bb-a20e-d334036748ad","commitTimeStamp":"2016-01-10T00:07:17.9972902Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1285.json","@type":"CatalogPage","commitId":"5a191db4-8574-461e-ae71-4ab031d4a1bc","commitTimeStamp":"2016-01-10T12:03:37.7831894Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1286.json","@type":"CatalogPage","commitId":"d66dffd5-32b5-4d05-aaf8-c029fa71a2b1","commitTimeStamp":"2016-01-10T21:43:58.951002Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1287.json","@type":"CatalogPage","commitId":"e9813358-d203-4b0d-985e-95f49af6c170","commitTimeStamp":"2016-01-11T07:44:17.1451Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1288.json","@type":"CatalogPage","commitId":"a2baa563-339c-4a04-9c1c-4c84ad95bc1b","commitTimeStamp":"2016-01-11T14:03:46.2439965Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1289.json","@type":"CatalogPage","commitId":"25a341ca-7c2c-4e7b-bc53-7ba7f0444448","commitTimeStamp":"2016-01-11T18:32:10.7486934Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1290.json","@type":"CatalogPage","commitId":"9fdb6d7b-97bb-4b87-9a71-aae4e40c7ec3","commitTimeStamp":"2016-01-12T01:33:31.5416717Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1291.json","@type":"CatalogPage","commitId":"69f7932c-22fe-4da6-b91c-af548f39f0ac","commitTimeStamp":"2016-01-12T08:27:31.7776428Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1292.json","@type":"CatalogPage","commitId":"da73ae5f-98bb-45b1-9dab-99969de5ae95","commitTimeStamp":"2016-01-12T14:06:46.8602011Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1293.json","@type":"CatalogPage","commitId":"68f75953-4297-49d6-92f3-5ea44b1119a1","commitTimeStamp":"2016-01-12T18:10:14.986442Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1294.json","@type":"CatalogPage","commitId":"7bc11ee8-8f42-4cd9-a0a6-32d532ed59cd","commitTimeStamp":"2016-01-13T02:03:59.3737099Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1295.json","@type":"CatalogPage","commitId":"92fafd09-cd8c-4d0f-b1b7-8db5fa3ca99d","commitTimeStamp":"2016-01-13T08:03:20.0554039Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page1296.json","@type":"CatalogPage","commitId":"6d81ecb4-145b-488c-898e-998ee577fc61","commitTimeStamp":"2016-01-13T10:31:00.7790306Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1297.json","@type":"CatalogPage","commitId":"ced44878-2591-4dae-a2dd-e0af89c6520c","commitTimeStamp":"2016-01-13T13:43:47.8605926Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1298.json","@type":"CatalogPage","commitId":"7a026857-9f36-461a-8367-3baaaf7b0ff4","commitTimeStamp":"2016-01-13T16:05:19.7480665Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1299.json","@type":"CatalogPage","commitId":"42ccf3cf-bdd4-4660-98a0-7e85b72e16bc","commitTimeStamp":"2016-01-13T18:32:49.4355024Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1300.json","@type":"CatalogPage","commitId":"7c51a2eb-dc87-40a8-831b-83469b2526d7","commitTimeStamp":"2016-01-13T22:11:37.7649356Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1301.json","@type":"CatalogPage","commitId":"e8ac41de-7e27-4800-8734-93bfc963b581","commitTimeStamp":"2016-01-14T02:11:36.8776109Z","count":558},{"@id":"https://api.nuget.org/v3/catalog0/page1302.json","@type":"CatalogPage","commitId":"40197b86-8476-49ef-a658-043d6ccf1faf","commitTimeStamp":"2016-01-14T06:04:46.4846191Z","count":553},{"@id":"https://api.nuget.org/v3/catalog0/page1303.json","@type":"CatalogPage","commitId":"242439f2-f5cb-4502-8d10-557434520f3d","commitTimeStamp":"2016-01-14T08:31:42.7203776Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1304.json","@type":"CatalogPage","commitId":"093d341b-f3d6-41d1-bb64-8f15c776ba00","commitTimeStamp":"2016-01-14T12:09:30.9757295Z","count":552},{"@id":"https://api.nuget.org/v3/catalog0/page1305.json","@type":"CatalogPage","commitId":"594c118a-cef0-4bb0-a768-faa81713b56e","commitTimeStamp":"2016-01-14T15:27:52.4861527Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page1306.json","@type":"CatalogPage","commitId":"4b65eb31-7e76-4d20-b31e-308593af4868","commitTimeStamp":"2016-01-14T18:11:07.4945951Z","count":552},{"@id":"https://api.nuget.org/v3/catalog0/page1307.json","@type":"CatalogPage","commitId":"eda5522b-ff40-4fdf-8eb9-92c360ba06a0","commitTimeStamp":"2016-01-14T22:05:00.4864953Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1308.json","@type":"CatalogPage","commitId":"fa026695-3f54-4a4d-9cdb-09cba26f7592","commitTimeStamp":"2016-01-15T01:37:40.565487Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1309.json","@type":"CatalogPage","commitId":"02842ef9-7e5c-4df5-9dad-fa09bf7675da","commitTimeStamp":"2016-01-15T04:02:48.8858301Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1310.json","@type":"CatalogPage","commitId":"58d8856f-51f1-45b8-ae9a-744ed275ba75","commitTimeStamp":"2016-01-15T08:05:02.7506195Z","count":552},{"@id":"https://api.nuget.org/v3/catalog0/page1311.json","@type":"CatalogPage","commitId":"56bf4d8e-047c-41ab-b698-c8113623eab9","commitTimeStamp":"2016-01-15T11:17:33.5429105Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1312.json","@type":"CatalogPage","commitId":"732ed97d-86fa-4f73-a5ab-8966986e6caf","commitTimeStamp":"2016-01-15T14:05:00.6271222Z","count":552},{"@id":"https://api.nuget.org/v3/catalog0/page1313.json","@type":"CatalogPage","commitId":"c652a5a3-a0ad-4986-bcf3-229aa33af5b0","commitTimeStamp":"2016-01-15T16:18:09.8452301Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1314.json","@type":"CatalogPage","commitId":"f362e4f0-e7b3-469b-9f63-5551a3a5ec3f","commitTimeStamp":"2016-01-15T20:08:09.150867Z","count":552},{"@id":"https://api.nuget.org/v3/catalog0/page1315.json","@type":"CatalogPage","commitId":"b9182d85-0396-4fcc-ab41-4f6f46048037","commitTimeStamp":"2016-01-16T00:08:17.7388537Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1316.json","@type":"CatalogPage","commitId":"ee67ddf0-a956-43d5-a869-6df3238a43f4","commitTimeStamp":"2016-01-16T03:38:34.7076983Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1317.json","@type":"CatalogPage","commitId":"8d7cb231-d0de-4a7e-a6e8-4bd820c422f6","commitTimeStamp":"2016-01-16T20:54:08.9489887Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1318.json","@type":"CatalogPage","commitId":"b3caaeab-0dc4-4503-b18c-1f0c7083bcd5","commitTimeStamp":"2016-01-17T18:06:33.7078505Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1319.json","@type":"CatalogPage","commitId":"2e55b690-d6c4-487c-8602-a2bf6ebd6ed7","commitTimeStamp":"2016-01-18T10:48:07.6713799Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1320.json","@type":"CatalogPage","commitId":"72533a57-2e77-44e2-b104-ca7e2c332c6b","commitTimeStamp":"2016-01-18T18:51:38.2775612Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1321.json","@type":"CatalogPage","commitId":"0e4fbdbd-34a1-47de-a3fd-a53bc5219ba5","commitTimeStamp":"2016-01-19T07:19:30.2224457Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1322.json","@type":"CatalogPage","commitId":"a31fbfd2-a0e0-4615-8d1f-04248fe4dc61","commitTimeStamp":"2016-01-19T14:26:16.3083829Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1323.json","@type":"CatalogPage","commitId":"7feb2410-c278-421b-abaf-8e5ce713beb2","commitTimeStamp":"2016-01-20T02:00:34.8868577Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1324.json","@type":"CatalogPage","commitId":"b2ab16e5-7ae0-4a86-a196-b5f18fe73820","commitTimeStamp":"2016-01-20T11:43:48.7771459Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1325.json","@type":"CatalogPage","commitId":"b4980306-be53-462f-9582-57cc9ebff7ad","commitTimeStamp":"2016-01-20T21:01:48.9635806Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1326.json","@type":"CatalogPage","commitId":"ab705804-8d0d-48c4-be24-04abecf82bd5","commitTimeStamp":"2016-01-21T14:30:06.0512904Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1327.json","@type":"CatalogPage","commitId":"43ac1ff5-a3dc-4d8a-8f3f-f9a5627f3a65","commitTimeStamp":"2016-01-22T00:57:58.8251016Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1328.json","@type":"CatalogPage","commitId":"902deb9d-f7ba-4ee8-a5e7-81b04ac1e11b","commitTimeStamp":"2016-01-22T13:45:54.964249Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1329.json","@type":"CatalogPage","commitId":"e41c091d-f2d6-4d9f-89ac-790f3730a87c","commitTimeStamp":"2016-01-23T02:09:29.7851893Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1330.json","@type":"CatalogPage","commitId":"0a4f8564-40a0-4faa-934e-4b273abc09b1","commitTimeStamp":"2016-01-23T18:00:15.3856108Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1331.json","@type":"CatalogPage","commitId":"cd382108-64a3-4d3a-ae9b-94c565882869","commitTimeStamp":"2016-01-24T17:31:40.9407434Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1332.json","@type":"CatalogPage","commitId":"a2ad5507-e351-4ab3-93e5-58e01a4f3334","commitTimeStamp":"2016-01-25T09:40:51.9382531Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1333.json","@type":"CatalogPage","commitId":"6a300530-1c10-4443-a522-4d2e1b3d51dc","commitTimeStamp":"2016-01-25T16:49:42.2188681Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1334.json","@type":"CatalogPage","commitId":"44e5b0cc-a5cc-485e-8dd5-599e38a0827f","commitTimeStamp":"2016-01-26T02:36:52.8718224Z","count":534},{"@id":"https://api.nuget.org/v3/catalog0/page1335.json","@type":"CatalogPage","commitId":"e114b218-b552-49df-8c69-c89481abc06f","commitTimeStamp":"2016-01-26T13:19:50.2755861Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1336.json","@type":"CatalogPage","commitId":"aac694c2-168f-42e2-84f5-3ce79e01910e","commitTimeStamp":"2016-01-27T04:10:20.8633167Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1337.json","@type":"CatalogPage","commitId":"1976602e-0ed8-4cf0-8a3b-f33c02785b46","commitTimeStamp":"2016-01-27T14:02:46.8557087Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1338.json","@type":"CatalogPage","commitId":"f13bb214-3748-44d4-8101-265731ad6539","commitTimeStamp":"2016-01-27T20:00:09.1195827Z","count":538},{"@id":"https://api.nuget.org/v3/catalog0/page1339.json","@type":"CatalogPage","commitId":"fc7ec1ee-314e-45a6-9399-c842e5a40f18","commitTimeStamp":"2016-01-27T20:05:53.3833994Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1340.json","@type":"CatalogPage","commitId":"c12f0995-9a44-43d2-b450-4a1c55751a67","commitTimeStamp":"2016-01-27T20:37:41.3205126Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1341.json","@type":"CatalogPage","commitId":"3c4036ce-ba5a-477a-85f4-69d0256aa537","commitTimeStamp":"2016-01-27T21:23:49.3776015Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1342.json","@type":"CatalogPage","commitId":"684897b6-4e4b-43be-816b-adc09c79ccc5","commitTimeStamp":"2016-01-27T22:36:59.3154202Z","count":541},{"@id":"https://api.nuget.org/v3/catalog0/page1343.json","@type":"CatalogPage","commitId":"70868067-b3fd-46eb-9c50-934266e6b242","commitTimeStamp":"2016-01-28T11:58:11.3566727Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1344.json","@type":"CatalogPage","commitId":"71749b4c-daf5-4d40-8f93-b429166ed780","commitTimeStamp":"2016-01-28T21:27:31.9699657Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1345.json","@type":"CatalogPage","commitId":"02132485-c40f-44d8-8e99-72eef6a0b940","commitTimeStamp":"2016-01-29T11:04:33.5768282Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1346.json","@type":"CatalogPage","commitId":"6172d8bd-0e36-461b-bcfe-2c1b8d2852d6","commitTimeStamp":"2016-01-30T07:23:09.8608817Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1347.json","@type":"CatalogPage","commitId":"1f15b520-8954-4ea7-a592-783f62864e25","commitTimeStamp":"2016-01-31T07:34:43.0607213Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1348.json","@type":"CatalogPage","commitId":"48055064-01c7-4250-9af9-eb92a9d388a6","commitTimeStamp":"2016-02-01T06:33:21.3939216Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1349.json","@type":"CatalogPage","commitId":"12734508-c3d0-4af1-8919-38d1f1e7dcf4","commitTimeStamp":"2016-02-01T14:37:04.0154318Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1350.json","@type":"CatalogPage","commitId":"7ee2c9ba-3422-4705-8bce-77f45032aef4","commitTimeStamp":"2016-02-02T01:05:18.083501Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1351.json","@type":"CatalogPage","commitId":"db758bb1-6fa9-41b0-8b7f-8132c610047b","commitTimeStamp":"2016-02-02T13:08:27.9296616Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1352.json","@type":"CatalogPage","commitId":"7d901406-18d9-49c6-a097-715ccca2577b","commitTimeStamp":"2016-02-02T22:39:42.1669353Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1353.json","@type":"CatalogPage","commitId":"2445e64a-1d7c-4362-922f-7e9ffed1ce6f","commitTimeStamp":"2016-02-03T13:00:46.3636001Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1354.json","@type":"CatalogPage","commitId":"ae76e6c4-d686-4a2f-8a46-13db1b4341d4","commitTimeStamp":"2016-02-03T20:35:03.7343201Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1355.json","@type":"CatalogPage","commitId":"1f7cef44-c599-4268-b312-8d1f71caf277","commitTimeStamp":"2016-02-04T11:41:03.786145Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1356.json","@type":"CatalogPage","commitId":"e0ff8353-e32f-4d0b-89f1-3b103c3d79c5","commitTimeStamp":"2016-02-04T20:41:21.8556985Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page1357.json","@type":"CatalogPage","commitId":"6e8cc376-aee5-42b0-a7ba-816e0687e34b","commitTimeStamp":"2016-02-05T08:39:50.0965094Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1358.json","@type":"CatalogPage","commitId":"bdcc3b5e-f4a3-4099-a6a5-96c58847734b","commitTimeStamp":"2016-02-05T19:22:16.8134966Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1359.json","@type":"CatalogPage","commitId":"518e9bd4-cd78-4c4f-a24c-27931ca19633","commitTimeStamp":"2016-02-06T16:53:48.8139677Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1360.json","@type":"CatalogPage","commitId":"44981589-8109-4093-a03a-068b386e7590","commitTimeStamp":"2016-02-07T19:35:24.8597104Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1361.json","@type":"CatalogPage","commitId":"320fe2c4-dc25-4e0d-956d-fed6cfdfc19e","commitTimeStamp":"2016-02-08T11:57:39.7085713Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1362.json","@type":"CatalogPage","commitId":"dfe157a4-1070-4cef-a9b4-a318a1f72760","commitTimeStamp":"2016-02-08T22:02:21.7504478Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1363.json","@type":"CatalogPage","commitId":"64041c53-4ad1-4fe4-82e2-489092bdb853","commitTimeStamp":"2016-02-09T11:36:58.4034291Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1364.json","@type":"CatalogPage","commitId":"bd390d09-9b54-4a2a-a4d7-99a0671e3312","commitTimeStamp":"2016-02-09T20:06:13.9234538Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1365.json","@type":"CatalogPage","commitId":"ac2c8599-d729-4bb5-99dd-bb443a35fd8e","commitTimeStamp":"2016-02-10T05:41:18.2258587Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1366.json","@type":"CatalogPage","commitId":"3b0b9b48-a0ef-4c38-ad53-5136657f0f06","commitTimeStamp":"2016-02-10T19:44:56.856772Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1367.json","@type":"CatalogPage","commitId":"a6fb5dc0-ab0d-4d6d-9f66-65f1c7b0e188","commitTimeStamp":"2016-02-11T10:00:49.8705942Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1368.json","@type":"CatalogPage","commitId":"a5786f68-c791-4000-a474-cc78154abaa6","commitTimeStamp":"2016-02-11T15:08:41.1201889Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1369.json","@type":"CatalogPage","commitId":"eaf5358b-e3c1-424b-9490-aebc0b05d230","commitTimeStamp":"2016-02-11T20:42:53.2699024Z","count":552},{"@id":"https://api.nuget.org/v3/catalog0/page1370.json","@type":"CatalogPage","commitId":"361e1fbf-8c95-42d8-8e19-2df95a8bb0c3","commitTimeStamp":"2016-02-12T10:34:37.710404Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page1371.json","@type":"CatalogPage","commitId":"97fc925b-9bdf-4be1-82a6-8166fe111e52","commitTimeStamp":"2016-02-12T15:20:08.3108985Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1372.json","@type":"CatalogPage","commitId":"2d3dafef-de6a-4410-9e7e-cde3fee00d9f","commitTimeStamp":"2016-02-12T21:15:18.168463Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page1373.json","@type":"CatalogPage","commitId":"9e8d7fee-c322-44e3-9311-297bb2f87c6e","commitTimeStamp":"2016-02-13T09:22:22.1803152Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1374.json","@type":"CatalogPage","commitId":"82ce2178-60b6-43ba-ad9c-ada099d53a81","commitTimeStamp":"2016-02-13T19:23:28.2725497Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page1375.json","@type":"CatalogPage","commitId":"63bfca67-25a5-4783-93e0-e9719d74e680","commitTimeStamp":"2016-02-14T16:00:36.8262208Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1376.json","@type":"CatalogPage","commitId":"27bfd703-0a7c-4be8-8068-0d622f4183b6","commitTimeStamp":"2016-02-15T08:46:45.0985488Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1377.json","@type":"CatalogPage","commitId":"3bfc8679-418d-42c5-a139-a4f7870b875e","commitTimeStamp":"2016-02-15T18:14:54.4525776Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1378.json","@type":"CatalogPage","commitId":"dc9b9ada-af3f-42e3-8cb1-fda99cae3a74","commitTimeStamp":"2016-02-16T01:33:18.0983359Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1379.json","@type":"CatalogPage","commitId":"2a742de1-f36c-4e54-b84c-01b915238d13","commitTimeStamp":"2016-02-16T04:46:43.112008Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1380.json","@type":"CatalogPage","commitId":"b5f722a9-75cf-4dc5-964f-0be6942ae213","commitTimeStamp":"2016-02-16T08:24:43.0452105Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1381.json","@type":"CatalogPage","commitId":"4b27d9b2-5cd4-44f0-b86c-fc99895785ca","commitTimeStamp":"2016-02-16T12:06:05.1335906Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1382.json","@type":"CatalogPage","commitId":"dd42c4c2-1b5e-4c5a-9618-4c9582428fb8","commitTimeStamp":"2016-02-16T14:34:51.2440288Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1383.json","@type":"CatalogPage","commitId":"c076ec59-b52a-4868-922b-2326d3302334","commitTimeStamp":"2016-02-16T18:18:39.7429849Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1384.json","@type":"CatalogPage","commitId":"92f8abe2-f86b-4590-a549-75f70811b19c","commitTimeStamp":"2016-02-16T22:08:15.0862655Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1385.json","@type":"CatalogPage","commitId":"d38a61ae-3033-40ac-acc4-a3eb9a22af53","commitTimeStamp":"2016-02-17T01:59:36.50162Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1386.json","@type":"CatalogPage","commitId":"58014dd5-75d6-4a94-837a-10fd1c23a132","commitTimeStamp":"2016-02-17T14:51:33.1591534Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1387.json","@type":"CatalogPage","commitId":"e94cea9b-4a9c-4ebe-8e7c-c6898981feef","commitTimeStamp":"2016-02-18T04:38:07.908071Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1388.json","@type":"CatalogPage","commitId":"bebabe45-722d-4e6c-ab03-70813c59e028","commitTimeStamp":"2016-02-18T16:46:34.3643182Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1389.json","@type":"CatalogPage","commitId":"e144559e-c220-4846-8ea4-9fe9f1fec11c","commitTimeStamp":"2016-02-19T05:13:52.1167422Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1390.json","@type":"CatalogPage","commitId":"de50b8af-d8e9-4303-a999-5ad616d12dda","commitTimeStamp":"2016-02-19T14:25:37.7120568Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1391.json","@type":"CatalogPage","commitId":"4bf613aa-8b3d-475a-984b-c4f3b4df2c95","commitTimeStamp":"2016-02-19T19:20:42.5269399Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1392.json","@type":"CatalogPage","commitId":"3f9e319c-cc38-4030-a8e0-7ccf041c8309","commitTimeStamp":"2016-02-19T19:46:28.6211936Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1393.json","@type":"CatalogPage","commitId":"689535f0-a8f0-4103-b9ca-47727dd3f9d4","commitTimeStamp":"2016-02-20T14:04:27.0211067Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1394.json","@type":"CatalogPage","commitId":"40d575c4-814e-420a-a8d0-35aa61be4dce","commitTimeStamp":"2016-02-21T11:15:06.5003771Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1395.json","@type":"CatalogPage","commitId":"6db2ee26-951f-4de3-a068-ea51b5319359","commitTimeStamp":"2016-02-22T10:06:47.3370524Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1396.json","@type":"CatalogPage","commitId":"f93c04ef-1acb-4cb0-b7a8-82f0e603032a","commitTimeStamp":"2016-02-22T17:17:15.2447378Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1397.json","@type":"CatalogPage","commitId":"de521600-9ca2-445d-98e5-5aef355d9afd","commitTimeStamp":"2016-02-23T00:09:14.4875439Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1398.json","@type":"CatalogPage","commitId":"4c06ec46-4284-4218-ae19-6045bf549894","commitTimeStamp":"2016-02-23T09:31:20.8655229Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1399.json","@type":"CatalogPage","commitId":"53dfbd6f-ea3a-4778-88eb-e96544729e6a","commitTimeStamp":"2016-02-23T11:17:38.2798217Z","count":532},{"@id":"https://api.nuget.org/v3/catalog0/page1400.json","@type":"CatalogPage","commitId":"fb58d9a3-eb92-4242-8fba-700972693ce9","commitTimeStamp":"2016-02-23T14:18:35.3914606Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page1401.json","@type":"CatalogPage","commitId":"02c471e6-6db8-430d-af32-5e15ebacf7a6","commitTimeStamp":"2016-02-23T21:37:36.6551601Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1402.json","@type":"CatalogPage","commitId":"60f34bbe-b21d-4285-aec1-546d619a03aa","commitTimeStamp":"2016-02-24T10:38:23.5558946Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1403.json","@type":"CatalogPage","commitId":"93e2a853-1275-4597-8e10-af5734906d21","commitTimeStamp":"2016-02-24T13:11:09.1462568Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1404.json","@type":"CatalogPage","commitId":"66b39952-e29e-4e3b-b81f-5bdc800bb7ed","commitTimeStamp":"2016-02-24T21:15:07.0726738Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1405.json","@type":"CatalogPage","commitId":"8339559f-fe16-4359-8aea-5dd918587351","commitTimeStamp":"2016-02-25T13:39:32.6241744Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1406.json","@type":"CatalogPage","commitId":"2fc696c7-5bab-4e3e-baf1-f31e3f64eef5","commitTimeStamp":"2016-02-25T22:44:36.748727Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1407.json","@type":"CatalogPage","commitId":"72331833-c498-4b59-b341-f708b68e526b","commitTimeStamp":"2016-02-26T12:06:27.8421238Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1408.json","@type":"CatalogPage","commitId":"53da939e-b3a6-434a-8af7-b22eea6dadbb","commitTimeStamp":"2016-02-26T20:35:26.4286597Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1409.json","@type":"CatalogPage","commitId":"d327ea3e-2b43-4c1d-bb3f-539c89a8c6cf","commitTimeStamp":"2016-02-27T15:00:24.2495234Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1410.json","@type":"CatalogPage","commitId":"dea68f23-bb85-4a90-929c-d64c237ed106","commitTimeStamp":"2016-02-28T15:12:50.9492391Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1411.json","@type":"CatalogPage","commitId":"5aa59fcc-da46-43d4-8137-f46a379238d1","commitTimeStamp":"2016-02-29T06:56:41.4661087Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1412.json","@type":"CatalogPage","commitId":"9430fdec-72f1-4892-a629-d291be0efeee","commitTimeStamp":"2016-02-29T14:46:37.1884272Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1413.json","@type":"CatalogPage","commitId":"92c5c418-eccc-4c77-ac47-63170abc5f10","commitTimeStamp":"2016-03-01T00:45:19.0590133Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1414.json","@type":"CatalogPage","commitId":"06300425-1e3b-44c0-b445-9f21fbe0c12d","commitTimeStamp":"2016-03-01T12:59:00.2272856Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1415.json","@type":"CatalogPage","commitId":"890c0a10-8234-4d44-8068-6538186a57d5","commitTimeStamp":"2016-03-02T00:02:46.1647747Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1416.json","@type":"CatalogPage","commitId":"1fbf035f-c88e-42c3-8dfe-657c53889803","commitTimeStamp":"2016-03-02T13:41:24.9395451Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1417.json","@type":"CatalogPage","commitId":"87444baf-0314-4a62-8309-4f754b617587","commitTimeStamp":"2016-03-03T03:41:35.63136Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1418.json","@type":"CatalogPage","commitId":"cc4575f8-17b5-4097-ab39-4bf53ea1c0fc","commitTimeStamp":"2016-03-03T16:31:20.6060866Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1419.json","@type":"CatalogPage","commitId":"16a6753d-0ded-41ac-b42b-551d59f872f4","commitTimeStamp":"2016-03-04T02:56:34.4484593Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1420.json","@type":"CatalogPage","commitId":"70498be2-fb6a-4126-859d-1dd920200f13","commitTimeStamp":"2016-03-04T14:40:47.0437114Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1421.json","@type":"CatalogPage","commitId":"1f508d0c-3b20-4b5d-897f-bfa462abc630","commitTimeStamp":"2016-03-05T08:43:32.4100258Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1422.json","@type":"CatalogPage","commitId":"44d06689-944d-4b07-96c3-3181f87272de","commitTimeStamp":"2016-03-06T08:43:25.6256398Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1423.json","@type":"CatalogPage","commitId":"137e00fa-502e-493a-b9d3-5e49a5bca4d6","commitTimeStamp":"2016-03-07T04:02:26.5421856Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1424.json","@type":"CatalogPage","commitId":"ba066d48-8b83-40a9-befe-1a6d14c9677f","commitTimeStamp":"2016-03-07T13:05:11.4603955Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1425.json","@type":"CatalogPage","commitId":"a89c946f-3e51-41fd-8cc0-c2226033971f","commitTimeStamp":"2016-03-08T01:05:13.5433694Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1426.json","@type":"CatalogPage","commitId":"b5e2820c-318f-4cb6-ac65-1a8868d906c7","commitTimeStamp":"2016-03-08T16:06:28.0239259Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1427.json","@type":"CatalogPage","commitId":"eb6a6f0c-94e6-4f4e-bd7d-2b7c7d0cb7d4","commitTimeStamp":"2016-03-09T04:31:54.9932574Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1428.json","@type":"CatalogPage","commitId":"11761eb7-c907-48c0-98b3-c79c305a61a3","commitTimeStamp":"2016-03-09T15:42:49.3052024Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1429.json","@type":"CatalogPage","commitId":"d48eeaca-939e-4a57-9a3c-d97742b67bcc","commitTimeStamp":"2016-03-10T05:23:21.3522424Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1430.json","@type":"CatalogPage","commitId":"04f159d8-072e-4a96-911a-431784226c29","commitTimeStamp":"2016-03-10T14:27:34.3396011Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page1431.json","@type":"CatalogPage","commitId":"93be802e-cecd-433d-b965-14822a18e557","commitTimeStamp":"2016-03-10T18:58:43.5502984Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1432.json","@type":"CatalogPage","commitId":"7266516a-ae3e-44a2-b9f6-95feafde0526","commitTimeStamp":"2016-03-11T03:06:17.3431199Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1433.json","@type":"CatalogPage","commitId":"664ed863-3b21-44e8-99d9-2248d19f6acf","commitTimeStamp":"2016-03-11T11:16:03.5141239Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1434.json","@type":"CatalogPage","commitId":"49d3b311-ebab-4291-b40b-17568cf04a60","commitTimeStamp":"2016-03-11T14:34:54.201487Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1435.json","@type":"CatalogPage","commitId":"4c7311df-9dcb-463c-a591-13b6bb6121b9","commitTimeStamp":"2016-03-11T20:11:29.7797882Z","count":551},{"@id":"https://api.nuget.org/v3/catalog0/page1436.json","@type":"CatalogPage","commitId":"bb0c5083-a8cd-4bba-b49e-d877b58bf853","commitTimeStamp":"2016-03-12T07:06:56.3828123Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1437.json","@type":"CatalogPage","commitId":"eda08ced-fc06-48a8-95d5-f03dd926ba3b","commitTimeStamp":"2016-03-12T21:43:27.7181834Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1438.json","@type":"CatalogPage","commitId":"f44a24ed-aa33-463c-913a-e3461f4535e1","commitTimeStamp":"2016-03-13T13:59:41.6972768Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1439.json","@type":"CatalogPage","commitId":"56d8e536-084c-4cbf-842d-3f0bdcf02d6e","commitTimeStamp":"2016-03-14T07:48:08.5576675Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1440.json","@type":"CatalogPage","commitId":"e0488fc4-1a32-4364-b644-d4e1e7b613fb","commitTimeStamp":"2016-03-14T15:57:49.7123039Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1441.json","@type":"CatalogPage","commitId":"aabbb258-6746-4980-8a96-27cd21fcd976","commitTimeStamp":"2016-03-14T23:14:40.562313Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1442.json","@type":"CatalogPage","commitId":"7f4600e6-7d8c-4a2f-bd08-46e9e6b6904e","commitTimeStamp":"2016-03-15T11:03:32.5052728Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1443.json","@type":"CatalogPage","commitId":"11a742a5-1d72-4fff-9bd2-880980d3f2fa","commitTimeStamp":"2016-03-15T17:45:27.4427774Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1444.json","@type":"CatalogPage","commitId":"7aa84b52-3f7f-42f3-af93-899ff52d883a","commitTimeStamp":"2016-03-16T09:01:09.5760281Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page1445.json","@type":"CatalogPage","commitId":"de584214-45e9-4ef2-ac32-ec0ff1dbb26f","commitTimeStamp":"2016-03-16T18:12:01.6349464Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1446.json","@type":"CatalogPage","commitId":"e5c78f9f-34ed-46d7-869b-784d159dbe3f","commitTimeStamp":"2016-03-17T05:31:12.7510662Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page1447.json","@type":"CatalogPage","commitId":"5cf2042c-510d-4833-b33d-36e3373f9cd0","commitTimeStamp":"2016-03-17T12:23:44.5287465Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1448.json","@type":"CatalogPage","commitId":"aa22b8e4-211a-438d-aaf9-fc1b2a20b5df","commitTimeStamp":"2016-03-17T14:15:26.771889Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1449.json","@type":"CatalogPage","commitId":"25d1e27a-0a47-4842-bfbd-a92813e56ba8","commitTimeStamp":"2016-03-17T16:14:17.3100559Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1450.json","@type":"CatalogPage","commitId":"b2158236-9e53-4bc7-aa74-3511b34cb78c","commitTimeStamp":"2016-03-17T18:09:28.0464878Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1451.json","@type":"CatalogPage","commitId":"e9260dbf-2656-423c-9cac-2da682635854","commitTimeStamp":"2016-03-17T20:07:46.5099026Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1452.json","@type":"CatalogPage","commitId":"80b47892-78a5-4efa-b481-a8e6acc5d7ac","commitTimeStamp":"2016-03-17T22:04:22.4330178Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1453.json","@type":"CatalogPage","commitId":"c8879eef-e12e-40c7-8a90-02d38465482b","commitTimeStamp":"2016-03-17T22:58:15.1975522Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1454.json","@type":"CatalogPage","commitId":"7092b25d-36b6-4111-9f21-b2206415c1ab","commitTimeStamp":"2016-03-18T01:39:03.0682349Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1455.json","@type":"CatalogPage","commitId":"3137957c-f6fb-4f9b-8dd4-ec2494ab4948","commitTimeStamp":"2016-03-18T04:06:02.9556258Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1456.json","@type":"CatalogPage","commitId":"5ba82b56-1abc-4016-a702-5f3bb264530b","commitTimeStamp":"2016-03-18T06:16:36.4504348Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1457.json","@type":"CatalogPage","commitId":"1bc72939-c8da-4689-8231-0bab856f689a","commitTimeStamp":"2016-03-18T08:20:42.8284943Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1458.json","@type":"CatalogPage","commitId":"40602af3-1b1d-4c59-a500-6474a05c7954","commitTimeStamp":"2016-03-18T10:21:26.9198183Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1459.json","@type":"CatalogPage","commitId":"966c9662-a4ff-4c53-8119-b554f11cd803","commitTimeStamp":"2016-03-18T12:18:06.0702198Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1460.json","@type":"CatalogPage","commitId":"c93fffe0-a53f-4297-bc20-fc15536d3d2f","commitTimeStamp":"2016-03-18T14:19:56.6838428Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1461.json","@type":"CatalogPage","commitId":"4cf0d192-d721-4ace-b2f3-b114ab10d7a1","commitTimeStamp":"2016-03-18T16:17:38.7991174Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1462.json","@type":"CatalogPage","commitId":"033cb340-4d33-49df-9f95-38bc589230c1","commitTimeStamp":"2016-03-18T18:21:20.2185116Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1463.json","@type":"CatalogPage","commitId":"b8135c41-b9e3-4447-a98e-9d675f535bf0","commitTimeStamp":"2016-03-18T20:24:33.9326399Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1464.json","@type":"CatalogPage","commitId":"51290751-4790-40a6-9275-fc909e89ca6e","commitTimeStamp":"2016-03-18T22:27:54.2814458Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1465.json","@type":"CatalogPage","commitId":"f3bdc671-66ad-490c-911b-92b6df60e172","commitTimeStamp":"2016-03-19T00:34:43.4614413Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1466.json","@type":"CatalogPage","commitId":"5b4dbb97-1f02-46fd-b812-ebd456305770","commitTimeStamp":"2016-03-19T02:45:33.8841262Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1467.json","@type":"CatalogPage","commitId":"fdf32583-bd36-45de-ba87-6b39e62e8448","commitTimeStamp":"2016-03-19T04:59:30.1403454Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1468.json","@type":"CatalogPage","commitId":"51717fbc-4836-46c8-be50-47e9dc551de6","commitTimeStamp":"2016-03-19T07:49:14.8549487Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1469.json","@type":"CatalogPage","commitId":"518732fe-9ee9-4dfc-bdc6-ed29d202d99a","commitTimeStamp":"2016-03-19T09:01:01.1781953Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1470.json","@type":"CatalogPage","commitId":"1243f062-89cd-450d-9754-33e6f6969052","commitTimeStamp":"2016-03-19T10:49:35.9050671Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1471.json","@type":"CatalogPage","commitId":"bd0e5965-943f-4372-a472-1d5671e798c3","commitTimeStamp":"2016-03-19T12:49:43.8773315Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1472.json","@type":"CatalogPage","commitId":"3e5bcf62-22b9-46b7-b256-20f754676c59","commitTimeStamp":"2016-03-19T14:58:14.5617517Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1473.json","@type":"CatalogPage","commitId":"4d30571b-7449-4a3f-9ae5-882d152804e4","commitTimeStamp":"2016-03-19T16:52:36.9071786Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1474.json","@type":"CatalogPage","commitId":"c9cda541-2a56-4ae0-828d-d33a3f8e1664","commitTimeStamp":"2016-03-19T18:46:40.8294973Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1475.json","@type":"CatalogPage","commitId":"94747338-b91e-411f-b3a2-191d19fa1e7f","commitTimeStamp":"2016-03-19T20:41:45.7467532Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1476.json","@type":"CatalogPage","commitId":"343cc976-81b6-44f9-82e7-87164249d470","commitTimeStamp":"2016-03-19T22:35:17.0019975Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1477.json","@type":"CatalogPage","commitId":"62427547-e41b-41e9-83b1-d26b21f99171","commitTimeStamp":"2016-03-20T00:42:18.4495836Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1478.json","@type":"CatalogPage","commitId":"c6d32d90-6f71-4fcf-80f7-c85faa2262ff","commitTimeStamp":"2016-03-20T02:53:55.8794081Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1479.json","@type":"CatalogPage","commitId":"f5d0d85b-e0a0-4005-b2f8-9c8b73f46b57","commitTimeStamp":"2016-03-20T05:53:02.4154575Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1480.json","@type":"CatalogPage","commitId":"08f4a874-bc75-43c3-b6d5-13fa21ad94ea","commitTimeStamp":"2016-03-20T08:13:09.396444Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1481.json","@type":"CatalogPage","commitId":"cc6e4102-f64d-4a28-bea7-0914a1a92756","commitTimeStamp":"2016-03-20T10:11:19.1246531Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1482.json","@type":"CatalogPage","commitId":"d4ab785a-143b-4e78-a7e1-5b62da498e11","commitTimeStamp":"2016-03-20T12:21:47.6283442Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1483.json","@type":"CatalogPage","commitId":"ffc79c5c-d56e-46cb-86fc-a05380c648ef","commitTimeStamp":"2016-03-20T14:25:06.0362487Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1484.json","@type":"CatalogPage","commitId":"b143724e-e0e5-47f6-8b72-97df1a7e66ce","commitTimeStamp":"2016-03-20T16:21:50.5037528Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1485.json","@type":"CatalogPage","commitId":"c0f9eb91-56dc-4db7-9c5b-3c854c04eda5","commitTimeStamp":"2016-03-20T18:28:08.6605886Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1486.json","@type":"CatalogPage","commitId":"7d859188-126d-47ff-ab0a-2c5dee42dfb4","commitTimeStamp":"2016-03-20T20:31:27.7604871Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1487.json","@type":"CatalogPage","commitId":"51a8b518-b509-4b85-8f04-6e9c7927eb67","commitTimeStamp":"2016-03-21T06:43:23.7653498Z","count":534},{"@id":"https://api.nuget.org/v3/catalog0/page1488.json","@type":"CatalogPage","commitId":"2f578597-51ba-4d63-af69-37bef352457b","commitTimeStamp":"2016-03-21T06:50:24.5466377Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1489.json","@type":"CatalogPage","commitId":"56f7d458-6cb3-4a71-a756-bd176f7b5db9","commitTimeStamp":"2016-03-21T06:55:23.1472341Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1490.json","@type":"CatalogPage","commitId":"0832a80b-07eb-42f1-9c27-0be0fc873295","commitTimeStamp":"2016-03-21T07:00:51.3563675Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1491.json","@type":"CatalogPage","commitId":"3cf24ebd-3276-446a-9e7a-6c4b4244e760","commitTimeStamp":"2016-03-21T07:05:22.2541114Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1492.json","@type":"CatalogPage","commitId":"684cc388-4838-46a5-a147-b58bca1d6fd0","commitTimeStamp":"2016-03-21T08:04:54.2638069Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1493.json","@type":"CatalogPage","commitId":"d62781db-4485-434a-8c95-2529ff87ab77","commitTimeStamp":"2016-03-21T10:03:57.7001179Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1494.json","@type":"CatalogPage","commitId":"9502880c-e2ff-4634-80aa-60ecc881a56f","commitTimeStamp":"2016-03-21T11:01:22.3301342Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1495.json","@type":"CatalogPage","commitId":"6cfed8fe-9207-467a-8072-a0de1df6419f","commitTimeStamp":"2016-03-21T14:33:08.8996725Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1496.json","@type":"CatalogPage","commitId":"b9d355d7-3c9f-4d79-9ae5-72b17586c814","commitTimeStamp":"2016-03-21T16:12:53.6565529Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1497.json","@type":"CatalogPage","commitId":"0e01914e-edbd-4a02-a6cb-9678f340639d","commitTimeStamp":"2016-03-21T17:45:14.2160669Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1498.json","@type":"CatalogPage","commitId":"240ff55a-d526-4217-8cc4-88d96a9ba2c8","commitTimeStamp":"2016-03-21T19:16:08.866267Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1499.json","@type":"CatalogPage","commitId":"fb3d4c05-eae4-405b-9321-be45f2006c53","commitTimeStamp":"2016-03-21T20:53:30.0441742Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1500.json","@type":"CatalogPage","commitId":"2a106b69-51aa-400e-87e4-1a6eda654434","commitTimeStamp":"2016-03-21T22:41:38.0326995Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1501.json","@type":"CatalogPage","commitId":"b6b39564-1044-4de3-8c8d-daaae2398927","commitTimeStamp":"2016-03-22T00:33:41.7471631Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1502.json","@type":"CatalogPage","commitId":"a4dd3926-4a9e-4c5b-8680-4fde7ecda68b","commitTimeStamp":"2016-03-22T02:33:08.7868907Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1503.json","@type":"CatalogPage","commitId":"e64ee14b-6f41-4ecf-8623-f5b3fc7dfc9b","commitTimeStamp":"2016-03-22T04:27:27.0314495Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1504.json","@type":"CatalogPage","commitId":"7ef069e8-bf18-4749-92f6-f504239ffc1d","commitTimeStamp":"2016-03-22T06:44:05.1931284Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1505.json","@type":"CatalogPage","commitId":"874a1dc6-1fe6-4d90-b124-ec66fdc9bf3d","commitTimeStamp":"2016-03-22T08:33:08.8894279Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1506.json","@type":"CatalogPage","commitId":"007ed6d9-bc73-4487-bd97-9865c393745d","commitTimeStamp":"2016-03-22T10:22:45.8049697Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1507.json","@type":"CatalogPage","commitId":"a7be57c1-959f-43c1-84d8-51441dfd45fa","commitTimeStamp":"2016-03-22T12:21:03.2655196Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1508.json","@type":"CatalogPage","commitId":"7d9f6fb9-2d19-4ebd-a83d-26ad3f48b554","commitTimeStamp":"2016-03-22T14:24:12.2187893Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1509.json","@type":"CatalogPage","commitId":"cf9029fa-700d-48c8-9d8a-6b8b7de45aec","commitTimeStamp":"2016-03-22T16:08:09.0649913Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1510.json","@type":"CatalogPage","commitId":"c0afeb75-9f3b-4bb3-b99f-29bf1c4c4bc8","commitTimeStamp":"2016-03-22T16:58:33.5323588Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1511.json","@type":"CatalogPage","commitId":"87a18e07-d356-4b6c-ab3f-2d42eba119c7","commitTimeStamp":"2016-03-22T18:10:32.1522305Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1512.json","@type":"CatalogPage","commitId":"af5a3a51-677a-4d4d-8551-a684e947adc2","commitTimeStamp":"2016-03-22T19:09:04.3029532Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1513.json","@type":"CatalogPage","commitId":"08327828-8aea-4298-884f-46e5dd63ffaa","commitTimeStamp":"2016-03-22T20:00:31.4779363Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1514.json","@type":"CatalogPage","commitId":"43e4bd9d-b324-4ac9-9d3a-14515dc90faa","commitTimeStamp":"2016-03-22T22:50:26.4735134Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1515.json","@type":"CatalogPage","commitId":"74007d51-7eda-4620-93c5-f2e676c697d0","commitTimeStamp":"2016-03-23T14:05:27.2310864Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1516.json","@type":"CatalogPage","commitId":"af407287-5e37-40d5-84cf-bb0f95f9dc5b","commitTimeStamp":"2016-03-24T00:13:43.0609827Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1517.json","@type":"CatalogPage","commitId":"74f89e04-e9c8-4c78-a0b8-199f4f46d51b","commitTimeStamp":"2016-03-24T15:19:36.7813104Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1518.json","@type":"CatalogPage","commitId":"9b648f9b-0922-4864-8614-8e5328c17a87","commitTimeStamp":"2016-03-25T02:14:18.176766Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1519.json","@type":"CatalogPage","commitId":"61ab64e8-0aa5-4cab-a93a-617345cfb5d8","commitTimeStamp":"2016-03-25T17:46:06.8833754Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1520.json","@type":"CatalogPage","commitId":"d77252ce-cd1e-418a-baca-993b230528d9","commitTimeStamp":"2016-03-26T15:15:03.6015729Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page1521.json","@type":"CatalogPage","commitId":"10c792d8-1ab9-42bd-9eeb-9d294a68ecf0","commitTimeStamp":"2016-03-27T12:42:15.2561141Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1522.json","@type":"CatalogPage","commitId":"63722049-d0c5-45ac-b014-1c4a36648520","commitTimeStamp":"2016-03-28T04:38:38.0259954Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page1523.json","@type":"CatalogPage","commitId":"94b966d5-0d24-43b2-b8c7-a39f70a1376c","commitTimeStamp":"2016-03-28T18:50:55.0657055Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1524.json","@type":"CatalogPage","commitId":"95258000-f861-48ef-a66c-b227dc5ae2cf","commitTimeStamp":"2016-03-29T08:59:14.5028272Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1525.json","@type":"CatalogPage","commitId":"588127c1-176e-425e-bbcb-35a44ad81176","commitTimeStamp":"2016-03-29T16:05:27.2706928Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1526.json","@type":"CatalogPage","commitId":"0019539d-c717-495e-b6f0-59da49228dd8","commitTimeStamp":"2016-03-30T02:47:54.8015383Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1527.json","@type":"CatalogPage","commitId":"414b90fd-a638-4863-9770-dc6334d1f491","commitTimeStamp":"2016-03-30T12:00:22.484389Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1528.json","@type":"CatalogPage","commitId":"509dd4a8-a1bb-4de4-acd0-c36ae05a41da","commitTimeStamp":"2016-03-30T19:42:57.4144034Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1529.json","@type":"CatalogPage","commitId":"ca673a0c-2aca-470a-afe6-716967433430","commitTimeStamp":"2016-03-31T06:07:42.7233607Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1530.json","@type":"CatalogPage","commitId":"0c1d0c78-ac52-4ef7-8610-491ec6b7dfc6","commitTimeStamp":"2016-03-31T12:15:53.8211449Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1531.json","@type":"CatalogPage","commitId":"9b30defa-4be5-4e8a-a971-1844c7cbfd08","commitTimeStamp":"2016-03-31T20:11:06.0791019Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1532.json","@type":"CatalogPage","commitId":"af6cbf9f-de7c-4182-a76e-1a14d44ff38d","commitTimeStamp":"2016-04-01T07:34:26.6450677Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1533.json","@type":"CatalogPage","commitId":"706da934-4cf7-4461-b541-81257421fb2f","commitTimeStamp":"2016-04-01T16:49:24.2247303Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1534.json","@type":"CatalogPage","commitId":"2ca93582-5787-409a-ac8a-9799dd38a89e","commitTimeStamp":"2016-04-02T06:08:52.7826058Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1535.json","@type":"CatalogPage","commitId":"d2ff87a2-649b-4437-a4b9-e34c4cdc1005","commitTimeStamp":"2016-04-03T06:50:03.6922002Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1536.json","@type":"CatalogPage","commitId":"d3806563-8f3d-4a1c-8804-5c03f54b1e48","commitTimeStamp":"2016-04-04T02:52:05.8269897Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1537.json","@type":"CatalogPage","commitId":"6bd3e571-69fc-4dad-b315-960b6930b872","commitTimeStamp":"2016-04-04T17:14:41.5851499Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1538.json","@type":"CatalogPage","commitId":"4d2c5d31-b0ae-496c-a86d-9e6f9478a2e9","commitTimeStamp":"2016-04-05T07:42:49.8089356Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1539.json","@type":"CatalogPage","commitId":"19e1b735-32a3-4954-9bd7-a1c10aead037","commitTimeStamp":"2016-04-05T10:08:19.1265401Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1540.json","@type":"CatalogPage","commitId":"3e2e1fda-1b99-4df2-90a2-f29fbbde73a6","commitTimeStamp":"2016-04-05T10:20:50.9383382Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page1541.json","@type":"CatalogPage","commitId":"384709ad-017a-46f5-a7f0-8a7e560273ee","commitTimeStamp":"2016-04-05T10:33:42.1686514Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1542.json","@type":"CatalogPage","commitId":"29c5dc1e-2366-44ea-b478-7b00c8d78c35","commitTimeStamp":"2016-04-05T10:59:50.9559175Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1543.json","@type":"CatalogPage","commitId":"a1c74bf1-b4a0-4355-a4a8-b0f54a046cc7","commitTimeStamp":"2016-04-05T18:38:34.2200261Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1544.json","@type":"CatalogPage","commitId":"52e6c531-0d9f-4b62-a3ae-946a9b934233","commitTimeStamp":"2016-04-06T07:43:54.5289217Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1545.json","@type":"CatalogPage","commitId":"6d1baf1c-0958-4a30-9a6d-ad9bcae5f12b","commitTimeStamp":"2016-04-06T13:27:14.1310913Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1546.json","@type":"CatalogPage","commitId":"b8675e2d-9183-42f2-8c15-2d029be00791","commitTimeStamp":"2016-04-06T20:26:13.9849982Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page1547.json","@type":"CatalogPage","commitId":"99204183-05ef-4cf9-9ea8-1ff7ecc0d76e","commitTimeStamp":"2016-04-07T06:42:54.1466684Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1548.json","@type":"CatalogPage","commitId":"8c99fddc-57b3-4f73-a569-f858f182889c","commitTimeStamp":"2016-04-07T15:36:17.8004513Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1549.json","@type":"CatalogPage","commitId":"d911ad28-5e1a-4630-b015-7d9c7de8b575","commitTimeStamp":"2016-04-08T02:10:09.2861209Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1550.json","@type":"CatalogPage","commitId":"364a58b0-dd93-4d22-b4cc-079bae356c76","commitTimeStamp":"2016-04-08T09:03:21.5562134Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1551.json","@type":"CatalogPage","commitId":"7037c10f-adbd-4977-96c7-374ec5c83ad5","commitTimeStamp":"2016-04-08T13:33:48.0947422Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1552.json","@type":"CatalogPage","commitId":"88421f03-d040-457e-b2be-b167f15e7ef5","commitTimeStamp":"2016-04-09T05:13:49.0988079Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1553.json","@type":"CatalogPage","commitId":"a1fbe279-b970-4c4f-9c15-db9158f14ef2","commitTimeStamp":"2016-04-10T01:58:25.5159213Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1554.json","@type":"CatalogPage","commitId":"bb98fe43-ff50-4188-95d3-218f8a762981","commitTimeStamp":"2016-04-10T17:58:25.028462Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1555.json","@type":"CatalogPage","commitId":"da7af02d-a162-45d8-a270-4ef71bc5761b","commitTimeStamp":"2016-04-11T11:16:40.4678277Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1556.json","@type":"CatalogPage","commitId":"53b17508-2c98-4245-96f8-e4ac7235a570","commitTimeStamp":"2016-04-11T19:40:08.7622656Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1557.json","@type":"CatalogPage","commitId":"0772ecc8-40f4-4aaf-ab44-64c09f616120","commitTimeStamp":"2016-04-12T09:29:21.5982737Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1558.json","@type":"CatalogPage","commitId":"ed3fe593-5e64-4dfd-a759-2d2db6ab8642","commitTimeStamp":"2016-04-12T15:10:56.947293Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1559.json","@type":"CatalogPage","commitId":"7ccbf9f1-bdac-4853-b46c-d557bc78ff1a","commitTimeStamp":"2016-04-13T04:51:33.9487128Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1560.json","@type":"CatalogPage","commitId":"a8d4e25b-93b5-4645-9874-62de43159d6a","commitTimeStamp":"2016-04-13T17:36:44.4462448Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1561.json","@type":"CatalogPage","commitId":"f3ee9a19-2803-435a-8851-53d7555adc44","commitTimeStamp":"2016-04-14T02:51:53.7299641Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page1562.json","@type":"CatalogPage","commitId":"67e165d8-b628-4b0b-902f-d619b17aebce","commitTimeStamp":"2016-04-14T03:04:18.4800499Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1563.json","@type":"CatalogPage","commitId":"9cd6697c-34dd-4b3c-a39f-d0f0963cce9e","commitTimeStamp":"2016-04-14T10:34:18.7866594Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1564.json","@type":"CatalogPage","commitId":"4d81a9aa-50e3-4f03-9411-dfa6559894d3","commitTimeStamp":"2016-04-14T13:39:02.4035603Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1565.json","@type":"CatalogPage","commitId":"7080d65f-c905-45df-8bd9-28c729091600","commitTimeStamp":"2016-04-14T19:51:22.7431231Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1566.json","@type":"CatalogPage","commitId":"508ff259-3b8b-42ba-ab9d-9b7c7535ea07","commitTimeStamp":"2016-04-15T05:24:00.3795809Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1567.json","@type":"CatalogPage","commitId":"3560e158-25aa-4ec3-93ab-846621e594a0","commitTimeStamp":"2016-04-15T14:55:40.1150931Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1568.json","@type":"CatalogPage","commitId":"07e669de-15d9-4ff9-a34a-8b7c8aa4668d","commitTimeStamp":"2016-04-16T05:50:32.9719769Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1569.json","@type":"CatalogPage","commitId":"0debfa9a-59e7-4114-9f93-c0c319e474fb","commitTimeStamp":"2016-04-17T13:40:53.7612251Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1570.json","@type":"CatalogPage","commitId":"dc0c53f8-ea53-413f-838c-3dadec373f7c","commitTimeStamp":"2016-04-18T06:39:52.3056278Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1571.json","@type":"CatalogPage","commitId":"72317aae-fbc1-40d2-b41e-739865c09824","commitTimeStamp":"2016-04-18T15:44:16.3739789Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1572.json","@type":"CatalogPage","commitId":"1b239b54-6604-4a9e-a7f5-bd59972635bf","commitTimeStamp":"2016-04-19T01:56:12.3153333Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1573.json","@type":"CatalogPage","commitId":"084a75ec-1807-4b4f-bea1-7269a7a98afd","commitTimeStamp":"2016-04-19T12:01:51.3557012Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1574.json","@type":"CatalogPage","commitId":"0cc926a8-67d8-4c00-b25e-902ad9f6225d","commitTimeStamp":"2016-04-19T20:30:56.3182658Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1575.json","@type":"CatalogPage","commitId":"f9ea92c2-18a5-4092-b0f0-3394e41ec484","commitTimeStamp":"2016-04-20T08:51:42.1659476Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1576.json","@type":"CatalogPage","commitId":"90b3d687-3b04-4246-a9aa-0a240ebf8186","commitTimeStamp":"2016-04-20T15:43:43.1096458Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1577.json","@type":"CatalogPage","commitId":"1ddf2f47-7aca-487d-ac33-016023e8da30","commitTimeStamp":"2016-04-21T03:39:53.1939303Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1578.json","@type":"CatalogPage","commitId":"e36f1a7e-a124-479f-898e-65d9cf4d8f05","commitTimeStamp":"2016-04-21T14:53:59.9696078Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1579.json","@type":"CatalogPage","commitId":"f2436ef2-c4b7-4f2a-96b3-50aaac114966","commitTimeStamp":"2016-04-22T05:45:52.1502308Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1580.json","@type":"CatalogPage","commitId":"687b7111-9c15-4232-93c0-1d53bcfe3426","commitTimeStamp":"2016-04-22T15:19:29.6923282Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1581.json","@type":"CatalogPage","commitId":"c8ba2359-bbc4-402e-9a29-fefba18464d4","commitTimeStamp":"2016-04-23T05:31:08.4768546Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1582.json","@type":"CatalogPage","commitId":"d070ca25-cae9-46eb-9e02-a69eab257979","commitTimeStamp":"2016-04-24T05:04:01.2044257Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1583.json","@type":"CatalogPage","commitId":"9382c4b4-b888-42b9-96c2-743723bc550f","commitTimeStamp":"2016-04-25T02:42:18.4971399Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1584.json","@type":"CatalogPage","commitId":"1b4cd465-ed05-442b-aed1-8039524dc724","commitTimeStamp":"2016-04-25T14:45:38.7166379Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1585.json","@type":"CatalogPage","commitId":"bb4a38b5-348a-4cd3-b13a-addc4b73c830","commitTimeStamp":"2016-04-26T06:13:17.2237359Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1586.json","@type":"CatalogPage","commitId":"dbb65239-0b86-4d38-88a2-f09fa0137f98","commitTimeStamp":"2016-04-26T18:29:57.9381745Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1587.json","@type":"CatalogPage","commitId":"571e3ca9-db97-4765-9d06-e1c490846dcf","commitTimeStamp":"2016-04-27T07:07:37.3510033Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1588.json","@type":"CatalogPage","commitId":"803fbbc3-5ba5-43d5-9978-40da20171566","commitTimeStamp":"2016-04-27T17:48:03.6209729Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1589.json","@type":"CatalogPage","commitId":"ee3b3f95-8fb2-4b21-a773-7dce32f5fc2c","commitTimeStamp":"2016-04-28T05:34:39.8796636Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1590.json","@type":"CatalogPage","commitId":"08c86459-d7f3-46c3-9c4b-f22a1d584f36","commitTimeStamp":"2016-04-28T14:30:20.2244679Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1591.json","@type":"CatalogPage","commitId":"b3ca2dab-d304-465b-9690-af0d60c174f2","commitTimeStamp":"2016-04-29T06:55:24.028766Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1592.json","@type":"CatalogPage","commitId":"4464f64e-2688-443a-8039-507b27cc2916","commitTimeStamp":"2016-04-29T18:42:20.7527012Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1593.json","@type":"CatalogPage","commitId":"ff1c1909-5946-4924-ae3a-af91a008c7d0","commitTimeStamp":"2016-04-30T15:32:50.2148678Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1594.json","@type":"CatalogPage","commitId":"37305c46-028a-4ae3-8ae7-6b93d2b4c664","commitTimeStamp":"2016-05-01T13:29:13.5403603Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1595.json","@type":"CatalogPage","commitId":"36d8b543-8fd8-465e-a6b7-c283b25e6f5a","commitTimeStamp":"2016-05-02T03:41:31.0944294Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1596.json","@type":"CatalogPage","commitId":"4b47e728-b6c6-4d8b-ad18-f6044a8c7bf5","commitTimeStamp":"2016-05-02T18:37:55.0846353Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1597.json","@type":"CatalogPage","commitId":"cd058719-6995-4437-8261-b7104ec6b1ca","commitTimeStamp":"2016-05-03T10:28:32.6496829Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1598.json","@type":"CatalogPage","commitId":"92a248f7-67aa-4c1d-8c3c-338b3741fc0a","commitTimeStamp":"2016-05-03T21:30:44.0560785Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1599.json","@type":"CatalogPage","commitId":"ca5e3836-ad3b-4a40-818d-72666cebc8b3","commitTimeStamp":"2016-05-04T11:04:34.7615534Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1600.json","@type":"CatalogPage","commitId":"c9faaf1a-caed-4912-a66e-795ffa9da641","commitTimeStamp":"2016-05-04T20:22:37.4133995Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1601.json","@type":"CatalogPage","commitId":"ebbb1e4f-82f7-4549-bb0b-55fcfbfaeed2","commitTimeStamp":"2016-05-05T13:55:46.7226301Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1602.json","@type":"CatalogPage","commitId":"373da887-f79f-4acd-9bd0-434fcf532d5e","commitTimeStamp":"2016-05-06T00:51:42.9548112Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1603.json","@type":"CatalogPage","commitId":"4a1e1774-ab44-4bb8-aeaa-96af9a4b6b7c","commitTimeStamp":"2016-05-06T12:50:56.0705506Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1604.json","@type":"CatalogPage","commitId":"1af72ead-dc0c-4fef-8b55-c5e09f263bc1","commitTimeStamp":"2016-05-07T02:37:05.2198629Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1605.json","@type":"CatalogPage","commitId":"2c5fe4e0-0e10-491f-9ac7-a792d1ad159f","commitTimeStamp":"2016-05-08T08:01:47.3129173Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1606.json","@type":"CatalogPage","commitId":"25bad4f3-04d2-4a4e-89e7-452eca02c380","commitTimeStamp":"2016-05-09T08:52:18.9522998Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1607.json","@type":"CatalogPage","commitId":"bfdbd882-c5c2-45a7-860a-d5ab20942f58","commitTimeStamp":"2016-05-09T18:39:20.1071662Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1608.json","@type":"CatalogPage","commitId":"0b44c45c-0166-4929-8d2c-a1a889191558","commitTimeStamp":"2016-05-10T06:45:20.449251Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1609.json","@type":"CatalogPage","commitId":"422471e3-7424-42a8-8c84-a0f9f99e66db","commitTimeStamp":"2016-05-10T17:11:50.1945288Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1610.json","@type":"CatalogPage","commitId":"cee05d83-5df0-4f66-80f3-9746a20f69d6","commitTimeStamp":"2016-05-11T06:31:28.9550764Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1611.json","@type":"CatalogPage","commitId":"71a7a89b-cebc-420b-bf8f-f12671b57771","commitTimeStamp":"2016-05-11T19:53:41.1405607Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1612.json","@type":"CatalogPage","commitId":"bb2d08f3-3b0a-4c38-9ab4-30bfda7d180b","commitTimeStamp":"2016-05-12T11:58:08.1851804Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1613.json","@type":"CatalogPage","commitId":"e11aaba0-e97e-418b-bf36-6efd1743bfd1","commitTimeStamp":"2016-05-12T21:09:01.5325022Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1614.json","@type":"CatalogPage","commitId":"f7690839-d23c-4db9-9943-be882e2c46fa","commitTimeStamp":"2016-05-13T10:28:32.2521267Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1615.json","@type":"CatalogPage","commitId":"17ed14ba-92a4-4573-94e7-e380c2456771","commitTimeStamp":"2016-05-13T21:04:13.6173458Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1616.json","@type":"CatalogPage","commitId":"8e015b39-59bd-4331-9f1e-e5e36832c024","commitTimeStamp":"2016-05-14T19:43:43.5746868Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1617.json","@type":"CatalogPage","commitId":"437345fd-38f1-405f-a08f-5db43f92e1c6","commitTimeStamp":"2016-05-16T01:52:23.4080034Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1618.json","@type":"CatalogPage","commitId":"2601d9c2-d4a5-4e8e-9d56-af8286f592c6","commitTimeStamp":"2016-05-16T13:55:10.2020083Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1619.json","@type":"CatalogPage","commitId":"7646493c-5e98-4d40-89cb-bd8cbe917d21","commitTimeStamp":"2016-05-16T17:00:01.062252Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1620.json","@type":"CatalogPage","commitId":"50bda762-ff62-46d2-8eac-9e991fc5d5d1","commitTimeStamp":"2016-05-17T06:44:26.1771671Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1621.json","@type":"CatalogPage","commitId":"8208b313-4687-4527-aa85-78ec41fcbcda","commitTimeStamp":"2016-05-17T16:47:08.1496985Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1622.json","@type":"CatalogPage","commitId":"0bc21ad8-1572-47f4-8b34-dedd33dc25c8","commitTimeStamp":"2016-05-18T04:46:38.9729469Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1623.json","@type":"CatalogPage","commitId":"7515b725-00f4-41a4-9e8c-cfc50372e18a","commitTimeStamp":"2016-05-18T13:29:42.0704299Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1624.json","@type":"CatalogPage","commitId":"5eb2b885-133c-4ee4-bd7e-3051172acfc1","commitTimeStamp":"2016-05-18T23:00:00.5099276Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1625.json","@type":"CatalogPage","commitId":"ddcc43d9-25a2-4da5-a35f-49e719d8c578","commitTimeStamp":"2016-05-19T10:31:30.5070926Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1626.json","@type":"CatalogPage","commitId":"1383890f-ec02-4e85-ad0f-a0a78a979e46","commitTimeStamp":"2016-05-19T18:07:24.1520791Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1627.json","@type":"CatalogPage","commitId":"8a37b3dd-dc12-4d81-a8e0-dd1edadac490","commitTimeStamp":"2016-05-20T07:13:33.8874836Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1628.json","@type":"CatalogPage","commitId":"fe6efc3b-456f-4907-bb55-8fe8ef775f20","commitTimeStamp":"2016-05-20T17:30:26.5961233Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page1629.json","@type":"CatalogPage","commitId":"8fd8afe8-0f7e-4f0e-91a7-ec6e88d6e7ea","commitTimeStamp":"2016-05-21T10:13:13.2826736Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1630.json","@type":"CatalogPage","commitId":"451d8b0d-bfa3-4467-8bdb-3d6b64b0cca5","commitTimeStamp":"2016-05-22T08:42:59.8745618Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1631.json","@type":"CatalogPage","commitId":"b568d90d-2b38-4d0f-bf51-2d8208e3451f","commitTimeStamp":"2016-05-23T05:24:43.7663015Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1632.json","@type":"CatalogPage","commitId":"d031deb2-9470-4b75-93fa-f1b66256d00e","commitTimeStamp":"2016-05-23T15:42:15.2811541Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1633.json","@type":"CatalogPage","commitId":"6cb09171-a659-4675-9732-5693f28629f9","commitTimeStamp":"2016-05-24T01:38:36.3997089Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1634.json","@type":"CatalogPage","commitId":"2ac22f8d-34cf-4a24-86f1-802219a12fe2","commitTimeStamp":"2016-05-24T12:35:30.9861715Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1635.json","@type":"CatalogPage","commitId":"36f4f026-da96-4025-86d8-3958e78091ef","commitTimeStamp":"2016-05-24T20:21:18.903459Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1636.json","@type":"CatalogPage","commitId":"197dd853-3369-4bcf-bc10-74b87cb1e721","commitTimeStamp":"2016-05-25T10:26:12.5882365Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1637.json","@type":"CatalogPage","commitId":"b6d0fff7-5f8c-42ad-98d6-c0bc9ea860a5","commitTimeStamp":"2016-05-25T16:52:35.4624625Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1638.json","@type":"CatalogPage","commitId":"efa7d116-0c7a-4647-939b-e0cff6b4d519","commitTimeStamp":"2016-05-26T02:39:03.3780589Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1639.json","@type":"CatalogPage","commitId":"91938996-f527-4e80-bca4-1aa177aa3672","commitTimeStamp":"2016-05-26T14:31:19.5538762Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1640.json","@type":"CatalogPage","commitId":"5be4e698-a380-45d3-a172-e81bd8956ef0","commitTimeStamp":"2016-05-26T21:32:52.1560898Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1641.json","@type":"CatalogPage","commitId":"3b3b549a-ef84-4435-be24-5da3801f4942","commitTimeStamp":"2016-05-27T02:08:21.9120997Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1642.json","@type":"CatalogPage","commitId":"7a50e260-90b5-4b5f-9802-dd1f2baaccad","commitTimeStamp":"2016-05-27T09:07:35.4973845Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1643.json","@type":"CatalogPage","commitId":"c57bacfa-d9e7-4725-9fff-cab97b821485","commitTimeStamp":"2016-05-27T11:56:23.5850167Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1644.json","@type":"CatalogPage","commitId":"9460b3b8-e021-4cee-ba69-4f393e38d430","commitTimeStamp":"2016-05-27T13:15:35.3776471Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1645.json","@type":"CatalogPage","commitId":"79d70eb4-72d9-44df-8b2a-51c53b487864","commitTimeStamp":"2016-05-27T17:03:51.2449939Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1646.json","@type":"CatalogPage","commitId":"731d1773-671c-40d0-86d8-87854bfbdc2e","commitTimeStamp":"2016-05-28T15:36:14.9626913Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1647.json","@type":"CatalogPage","commitId":"7db179fc-7f8c-4126-8cb8-163b7838d7c9","commitTimeStamp":"2016-05-29T17:19:16.4358806Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1648.json","@type":"CatalogPage","commitId":"3f370cb3-73f8-4999-923f-4398a1971a91","commitTimeStamp":"2016-05-30T12:52:17.5600527Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1649.json","@type":"CatalogPage","commitId":"05c8fe61-b1c1-4fc5-af72-b922fb8b62db","commitTimeStamp":"2016-05-30T21:14:13.2088123Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1650.json","@type":"CatalogPage","commitId":"ac3b14df-74bc-40c3-922b-a199cfcb16af","commitTimeStamp":"2016-05-31T10:49:08.330607Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1651.json","@type":"CatalogPage","commitId":"1da6c4c1-3a5c-4dee-8c48-2fe0171f77b5","commitTimeStamp":"2016-05-31T18:05:36.0363896Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1652.json","@type":"CatalogPage","commitId":"76dd62a3-88a5-4dea-ad4f-61f9b7fb646a","commitTimeStamp":"2016-06-01T05:01:35.11999Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1653.json","@type":"CatalogPage","commitId":"3e704a99-b05f-44ff-8382-a5443c6a418e","commitTimeStamp":"2016-06-01T14:47:27.8178751Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1654.json","@type":"CatalogPage","commitId":"81f03397-a72b-44f7-9067-18cf0259cc59","commitTimeStamp":"2016-06-01T23:17:04.4833303Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1655.json","@type":"CatalogPage","commitId":"bbd5be0b-2373-440d-a1de-932cc706b08f","commitTimeStamp":"2016-06-02T10:03:22.1218366Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1656.json","@type":"CatalogPage","commitId":"237a5d78-816f-4f6b-a0f1-67e5affbfd39","commitTimeStamp":"2016-06-02T16:21:23.0578173Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1657.json","@type":"CatalogPage","commitId":"c767b34b-35a8-4ed0-a274-25f0e784db85","commitTimeStamp":"2016-06-02T23:59:04.832769Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1658.json","@type":"CatalogPage","commitId":"77fdff6b-78e8-4641-97e4-d1be5794da32","commitTimeStamp":"2016-06-03T14:29:20.2219993Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1659.json","@type":"CatalogPage","commitId":"4cb7f2a4-02ac-46d4-99a3-6370ba0e6bf5","commitTimeStamp":"2016-06-03T22:59:07.0500279Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1660.json","@type":"CatalogPage","commitId":"757854d0-b1bd-4778-9066-7e109ee96204","commitTimeStamp":"2016-06-04T20:39:42.3372059Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1661.json","@type":"CatalogPage","commitId":"d4ae88ab-5d95-422a-ace0-04293911fd85","commitTimeStamp":"2016-06-05T19:15:29.5951226Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1662.json","@type":"CatalogPage","commitId":"762ae9c6-0ce7-4e69-a368-5e91e275f2ae","commitTimeStamp":"2016-06-06T11:53:43.5304726Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1663.json","@type":"CatalogPage","commitId":"e294df2f-bc5e-4b74-8a34-97f60d1fd424","commitTimeStamp":"2016-06-06T19:57:36.4928518Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1664.json","@type":"CatalogPage","commitId":"97517e7d-96f4-450f-b154-4c18b746e6a9","commitTimeStamp":"2016-06-07T07:13:12.2759287Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1665.json","@type":"CatalogPage","commitId":"0f59022f-4fff-41b1-ac2e-da141e6553ff","commitTimeStamp":"2016-06-07T10:19:11.5998992Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1666.json","@type":"CatalogPage","commitId":"603f6911-2194-4020-a80e-e8ef2b748565","commitTimeStamp":"2016-06-07T18:09:20.6102619Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1667.json","@type":"CatalogPage","commitId":"e5c4aeb9-d724-4421-8817-143ceb002d6d","commitTimeStamp":"2016-06-08T08:43:48.9181295Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1668.json","@type":"CatalogPage","commitId":"c2e57b1b-25eb-4604-84d8-7e9f52d80689","commitTimeStamp":"2016-06-08T16:13:32.945719Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1669.json","@type":"CatalogPage","commitId":"ad70904f-31c7-49a9-8c7f-9bb843d6dbf2","commitTimeStamp":"2016-06-09T03:56:34.8503984Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1670.json","@type":"CatalogPage","commitId":"6624e541-ccd3-4a9c-bf79-5f7dc2898bbe","commitTimeStamp":"2016-06-09T14:14:36.5777674Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1671.json","@type":"CatalogPage","commitId":"412cbd0d-0f00-4890-a39a-f1cd924cef64","commitTimeStamp":"2016-06-09T23:31:31.4695657Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1672.json","@type":"CatalogPage","commitId":"f74f84d8-eb68-4d96-b9ef-aa9a35889758","commitTimeStamp":"2016-06-10T14:10:15.4649144Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1673.json","@type":"CatalogPage","commitId":"3f1800db-0320-4f06-80b6-3e067dde3e73","commitTimeStamp":"2016-06-11T03:03:04.5752717Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1674.json","@type":"CatalogPage","commitId":"9b66a583-5483-452f-a9f3-442b68180936","commitTimeStamp":"2016-06-11T19:38:51.1124053Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1675.json","@type":"CatalogPage","commitId":"f1d872ee-c21b-4fb1-8cec-665d31ed2ec8","commitTimeStamp":"2016-06-13T01:13:45.757257Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1676.json","@type":"CatalogPage","commitId":"47c36aaf-2629-49d9-8e76-19e6f245ddec","commitTimeStamp":"2016-06-13T13:06:29.8344151Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1677.json","@type":"CatalogPage","commitId":"ffae06c4-e4cc-4874-ac27-9d94f484e3b7","commitTimeStamp":"2016-06-14T00:17:15.4604851Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1678.json","@type":"CatalogPage","commitId":"0d5be0c1-679a-45e6-92cc-b5ad93945a04","commitTimeStamp":"2016-06-14T07:44:23.7057667Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1679.json","@type":"CatalogPage","commitId":"37d6b6fa-9d7a-4fa1-927a-b0832bf0d914","commitTimeStamp":"2016-06-14T14:13:16.2033765Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1680.json","@type":"CatalogPage","commitId":"1f6ac2b8-d517-4b50-82b6-f9d06e9f932c","commitTimeStamp":"2016-06-15T02:32:47.9987213Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1681.json","@type":"CatalogPage","commitId":"2ee16324-dc9e-4ff0-a906-44908200e42a","commitTimeStamp":"2016-06-15T11:35:39.8713347Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1682.json","@type":"CatalogPage","commitId":"bb510450-39fc-45d2-b39a-a9da5469cff9","commitTimeStamp":"2016-06-15T17:42:36.9018887Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page1683.json","@type":"CatalogPage","commitId":"0e19fa37-7940-4ec5-9697-0ad696415a7b","commitTimeStamp":"2016-06-16T08:13:24.3459409Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1684.json","@type":"CatalogPage","commitId":"0d3954fb-3494-47b2-b0b4-514bdd9f8f34","commitTimeStamp":"2016-06-16T15:22:24.8988531Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1685.json","@type":"CatalogPage","commitId":"604f0799-60c5-4f1a-816f-ad69d4803c98","commitTimeStamp":"2016-06-17T02:23:53.3136582Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1686.json","@type":"CatalogPage","commitId":"4810339e-0a63-4616-8273-97141341af7e","commitTimeStamp":"2016-06-17T13:02:02.9566948Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1687.json","@type":"CatalogPage","commitId":"2efa98b1-0378-4444-95a0-237ad47d2daf","commitTimeStamp":"2016-06-17T19:41:11.0031628Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1688.json","@type":"CatalogPage","commitId":"06cb873c-2651-4d9d-af2c-cf3475c4b1ec","commitTimeStamp":"2016-06-18T12:12:38.6451219Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1689.json","@type":"CatalogPage","commitId":"310fa210-755a-433a-9fde-4f2528851600","commitTimeStamp":"2016-06-19T04:05:46.631373Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1690.json","@type":"CatalogPage","commitId":"8b5146af-61d9-40d1-ac9f-ab06f79d8ad2","commitTimeStamp":"2016-06-19T17:54:42.0733638Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1691.json","@type":"CatalogPage","commitId":"d539ea5c-7e68-4b05-bba0-dc3fc728bbef","commitTimeStamp":"2016-06-20T10:10:30.7124845Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1692.json","@type":"CatalogPage","commitId":"c26e922b-7a4f-483d-bca5-831d532ef8e2","commitTimeStamp":"2016-06-20T17:24:41.6415323Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1693.json","@type":"CatalogPage","commitId":"671cbcaa-9085-4954-a2be-54d827d52683","commitTimeStamp":"2016-06-21T04:17:35.9190526Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1694.json","@type":"CatalogPage","commitId":"4422dedd-7232-4328-8685-53eed12a0987","commitTimeStamp":"2016-06-21T15:28:12.0067645Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1695.json","@type":"CatalogPage","commitId":"2c4da3ed-7dc9-40a5-90a9-49c3fd316484","commitTimeStamp":"2016-06-21T20:09:27.8751487Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1696.json","@type":"CatalogPage","commitId":"dc7f6082-5e36-430a-b6a4-847a61abc3f5","commitTimeStamp":"2016-06-22T09:58:53.1935698Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1697.json","@type":"CatalogPage","commitId":"1d969e42-868c-4bfe-93d3-9ac8ce037ba7","commitTimeStamp":"2016-06-22T19:12:46.9072246Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1698.json","@type":"CatalogPage","commitId":"3c25d9a8-7999-4139-a475-9ae6d39ca922","commitTimeStamp":"2016-06-23T05:19:15.0786014Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1699.json","@type":"CatalogPage","commitId":"f2e35c37-13eb-491d-a972-8f6b246e2d66","commitTimeStamp":"2016-06-23T16:06:19.6504656Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1700.json","@type":"CatalogPage","commitId":"67d4f71a-9247-4ff4-9682-7aa133d25333","commitTimeStamp":"2016-06-24T02:19:19.307689Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1701.json","@type":"CatalogPage","commitId":"0f2fa6d6-4eb4-425b-acf4-16f70e7ab9f2","commitTimeStamp":"2016-06-24T12:18:20.9042798Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1702.json","@type":"CatalogPage","commitId":"119e1d11-bfb7-4569-8134-ae9b99695082","commitTimeStamp":"2016-06-25T04:02:15.7759584Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1703.json","@type":"CatalogPage","commitId":"df69305d-e26c-4563-b14c-914a82b27a2f","commitTimeStamp":"2016-06-26T02:37:13.2711785Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1704.json","@type":"CatalogPage","commitId":"fc9c3995-0358-4126-822b-73f7d2a25fe5","commitTimeStamp":"2016-06-27T02:42:32.4065737Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1705.json","@type":"CatalogPage","commitId":"589fa558-616b-4ed7-94e5-6d0f09e73a21","commitTimeStamp":"2016-06-27T12:10:34.8031834Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1706.json","@type":"CatalogPage","commitId":"09bc40e6-bb2a-4631-89c3-df7e2d3a6b56","commitTimeStamp":"2016-06-27T14:29:41.618862Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1707.json","@type":"CatalogPage","commitId":"b10ff992-8727-4f87-8b70-6263f56fa937","commitTimeStamp":"2016-06-27T23:10:48.3907299Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1708.json","@type":"CatalogPage","commitId":"dc1e524b-8bdf-4f6f-a735-d19a1b7d057c","commitTimeStamp":"2016-06-28T08:02:40.495752Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1709.json","@type":"CatalogPage","commitId":"c6d68697-4d20-4ce0-bfb0-9c32e3dc3b74","commitTimeStamp":"2016-06-28T12:49:29.4239216Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1710.json","@type":"CatalogPage","commitId":"82d3a97c-9475-46aa-8d4e-12cc6d144c44","commitTimeStamp":"2016-06-28T18:47:27.9502628Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1711.json","@type":"CatalogPage","commitId":"2a7dbbf9-2049-4466-aac9-22b933a38a10","commitTimeStamp":"2016-06-29T00:29:10.6895024Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1712.json","@type":"CatalogPage","commitId":"87028d22-2d9f-4844-9f4f-dc881a07f556","commitTimeStamp":"2016-06-29T11:43:02.5635124Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1713.json","@type":"CatalogPage","commitId":"c67aa1c5-2230-4269-b329-9094ecfbdd15","commitTimeStamp":"2016-06-29T19:00:39.5912338Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1714.json","@type":"CatalogPage","commitId":"804b8ad0-f8cf-48db-a4a2-753b280c21a5","commitTimeStamp":"2016-06-30T06:07:02.944923Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1715.json","@type":"CatalogPage","commitId":"120afbea-be92-49a3-96c6-e83f27d75f89","commitTimeStamp":"2016-06-30T13:28:38.7583143Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1716.json","@type":"CatalogPage","commitId":"3a1b61c9-cd3a-4037-9e4b-f9812a6714f6","commitTimeStamp":"2016-06-30T22:28:03.0843567Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1717.json","@type":"CatalogPage","commitId":"19698e06-8e10-4f6c-bcc7-11289eb3ca36","commitTimeStamp":"2016-07-01T10:03:04.1767609Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1718.json","@type":"CatalogPage","commitId":"39a2778a-ff52-48c7-be93-18e50d890ad0","commitTimeStamp":"2016-07-01T14:39:17.3092665Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1719.json","@type":"CatalogPage","commitId":"b167cf7b-45fb-461a-8c15-4e9a82229b19","commitTimeStamp":"2016-07-01T23:41:18.8342608Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1720.json","@type":"CatalogPage","commitId":"d0b3d836-113e-4fa5-a5ca-5ad9fb28b9ee","commitTimeStamp":"2016-07-02T22:02:58.6449561Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1721.json","@type":"CatalogPage","commitId":"8f3a746e-629c-4fd9-8bfb-856098364d7e","commitTimeStamp":"2016-07-04T01:41:09.9836562Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1722.json","@type":"CatalogPage","commitId":"df77c6dd-f235-4477-814e-c28a8627dc51","commitTimeStamp":"2016-07-04T11:41:52.4794306Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1723.json","@type":"CatalogPage","commitId":"6fb58b6c-6986-4ddb-b8e5-bbec6395dc80","commitTimeStamp":"2016-07-04T22:38:06.923776Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1724.json","@type":"CatalogPage","commitId":"1a35caf7-3717-4808-a90c-92d46d194dea","commitTimeStamp":"2016-07-05T12:24:35.2587477Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1725.json","@type":"CatalogPage","commitId":"c64983a4-915f-4751-abf9-0961b9995b79","commitTimeStamp":"2016-07-05T17:07:14.4059421Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1726.json","@type":"CatalogPage","commitId":"8065c963-ede5-45ca-8274-1872f8d4d409","commitTimeStamp":"2016-07-06T04:18:13.3051533Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1727.json","@type":"CatalogPage","commitId":"5ca32592-bee2-4892-be5d-79f8a1444c86","commitTimeStamp":"2016-07-06T16:53:49.1692174Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1728.json","@type":"CatalogPage","commitId":"6a54da09-4a83-40fe-90da-75e655f29215","commitTimeStamp":"2016-07-07T07:09:06.698976Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1729.json","@type":"CatalogPage","commitId":"b6622710-5f8d-47b2-9c76-7689e02340e7","commitTimeStamp":"2016-07-07T14:17:27.8132595Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1730.json","@type":"CatalogPage","commitId":"d496e9bc-a1c0-424f-970d-bbbaa8153fd7","commitTimeStamp":"2016-07-07T22:18:16.124361Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1731.json","@type":"CatalogPage","commitId":"cd106858-7e9b-427d-bbf0-499bcf0be3fb","commitTimeStamp":"2016-07-08T11:36:45.1826221Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1732.json","@type":"CatalogPage","commitId":"f68a8bb0-7c7b-44d8-96bf-948135c29b04","commitTimeStamp":"2016-07-08T22:45:11.5323939Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1733.json","@type":"CatalogPage","commitId":"64e22f8c-452e-4760-aa74-8af39a87bc1c","commitTimeStamp":"2016-07-09T15:43:12.9733002Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1734.json","@type":"CatalogPage","commitId":"6c13d43d-e1d9-4544-8e7c-1600cd50ca81","commitTimeStamp":"2016-07-10T17:34:59.888075Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1735.json","@type":"CatalogPage","commitId":"b9bd942f-e798-4218-9078-61b236cc4265","commitTimeStamp":"2016-07-11T12:27:08.4165064Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1736.json","@type":"CatalogPage","commitId":"024a3281-3304-431f-8b75-22b87de68a30","commitTimeStamp":"2016-07-11T20:24:47.6853792Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1737.json","@type":"CatalogPage","commitId":"13154238-2c40-48d2-b6d2-9eb5e8beabd8","commitTimeStamp":"2016-07-12T10:02:44.2609887Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1738.json","@type":"CatalogPage","commitId":"c8a35fd0-9e39-4e27-9251-6f4143dbb4d4","commitTimeStamp":"2016-07-12T17:17:19.9061698Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1739.json","@type":"CatalogPage","commitId":"8557f9cc-86c1-410e-b0ab-ca5b50db6d62","commitTimeStamp":"2016-07-13T00:58:50.330643Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1740.json","@type":"CatalogPage","commitId":"18422798-1bda-44d4-af4f-2f91654fbcc6","commitTimeStamp":"2016-07-13T11:53:42.3665854Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1741.json","@type":"CatalogPage","commitId":"594e682e-1db6-42b6-8091-c77e689855c6","commitTimeStamp":"2016-07-13T23:10:27.8229444Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1742.json","@type":"CatalogPage","commitId":"da27f870-28e8-488c-b3d0-2f7a7264606d","commitTimeStamp":"2016-07-14T10:42:51.1083312Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1743.json","@type":"CatalogPage","commitId":"7fb94421-d4dd-4337-b305-4bfce557be2d","commitTimeStamp":"2016-07-14T16:29:47.2718964Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1744.json","@type":"CatalogPage","commitId":"8fb1a048-5aa1-400a-baa6-136d4221cfc1","commitTimeStamp":"2016-07-15T05:37:47.6295227Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1745.json","@type":"CatalogPage","commitId":"e3e20065-be12-47c0-89f7-7b3426ec6b1a","commitTimeStamp":"2016-07-15T08:36:38.97377Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page1746.json","@type":"CatalogPage","commitId":"6beb11c7-587a-4528-9211-de01e2c7877a","commitTimeStamp":"2016-07-15T08:52:12.1239891Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page1747.json","@type":"CatalogPage","commitId":"95dfa78f-5f85-4800-9a9c-d5e71627ec7a","commitTimeStamp":"2016-07-15T14:25:42.9406589Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1748.json","@type":"CatalogPage","commitId":"141efa59-c9bf-4df6-a5f2-89d192a22415","commitTimeStamp":"2016-07-16T01:14:49.6577789Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1749.json","@type":"CatalogPage","commitId":"2c7efd91-ddac-4e0b-9778-58414cc721d8","commitTimeStamp":"2016-07-16T15:26:04.0839423Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1750.json","@type":"CatalogPage","commitId":"09d8b12c-bcec-46ba-bdaf-046a1513e9e9","commitTimeStamp":"2016-07-17T07:20:13.7123308Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1751.json","@type":"CatalogPage","commitId":"2905d02e-c1b3-4e1a-99c9-c6a5a00c5edb","commitTimeStamp":"2016-07-17T18:32:51.8851378Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1752.json","@type":"CatalogPage","commitId":"f9f2f6c7-7e3d-4719-8ace-e1058bfcd434","commitTimeStamp":"2016-07-18T08:04:29.1198554Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1753.json","@type":"CatalogPage","commitId":"2ac7ba39-0caa-4244-a5a3-8386a1a6fcd3","commitTimeStamp":"2016-07-18T17:28:47.3568918Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1754.json","@type":"CatalogPage","commitId":"ae86e45d-d796-43d6-b2da-d4e324ef07fa","commitTimeStamp":"2016-07-18T23:54:55.7635844Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1755.json","@type":"CatalogPage","commitId":"3cd0b91c-7437-4558-a88a-0f025c4022e1","commitTimeStamp":"2016-07-19T10:26:45.3969585Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1756.json","@type":"CatalogPage","commitId":"818c88ce-9021-420a-a7ed-49629f7f2b93","commitTimeStamp":"2016-07-19T16:58:30.728871Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1757.json","@type":"CatalogPage","commitId":"41107feb-f350-47ac-810e-4f624d80c0b7","commitTimeStamp":"2016-07-20T03:25:09.4561105Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1758.json","@type":"CatalogPage","commitId":"76381921-9b91-4135-a52c-888eebaa745b","commitTimeStamp":"2016-07-20T12:57:42.8818527Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page1759.json","@type":"CatalogPage","commitId":"a78e311b-6d13-4692-8aa4-eb78e47ea0dd","commitTimeStamp":"2016-07-20T19:32:42.8575732Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1760.json","@type":"CatalogPage","commitId":"5c8cdec1-6306-4c9e-93dc-66f55620820c","commitTimeStamp":"2016-07-21T08:01:57.2762024Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1761.json","@type":"CatalogPage","commitId":"50e6376e-74de-469a-b8e6-b86d5e40f717","commitTimeStamp":"2016-07-21T14:04:08.9685609Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1762.json","@type":"CatalogPage","commitId":"22c04a5f-cf95-4453-8755-653d6ca0d0de","commitTimeStamp":"2016-07-21T18:16:19.1916024Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1763.json","@type":"CatalogPage","commitId":"43d50ae1-26ad-4298-a236-0100d77380a6","commitTimeStamp":"2016-07-22T01:05:32.7063086Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page1764.json","@type":"CatalogPage","commitId":"bb4d8d45-ea98-4828-b286-eedf89113722","commitTimeStamp":"2016-07-22T10:23:11.7501921Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1765.json","@type":"CatalogPage","commitId":"58b8bcb5-776e-4f46-915d-1760422e13e9","commitTimeStamp":"2016-07-22T19:13:31.6435498Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1766.json","@type":"CatalogPage","commitId":"8be06b9b-d685-4eab-b9dc-59aae51c4c88","commitTimeStamp":"2016-07-23T11:18:02.2078331Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1767.json","@type":"CatalogPage","commitId":"892273eb-dd02-4ca0-a4d1-8aa1d93af362","commitTimeStamp":"2016-07-24T11:16:26.3254083Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1768.json","@type":"CatalogPage","commitId":"3ec215ae-699f-4f93-b1cf-352ec18ba376","commitTimeStamp":"2016-07-25T07:48:37.025044Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1769.json","@type":"CatalogPage","commitId":"2000b7ea-a36f-46fd-941a-6d7c23941429","commitTimeStamp":"2016-07-25T18:22:09.2415573Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1770.json","@type":"CatalogPage","commitId":"e4b02c39-57dd-4571-8c4d-e9b193f44998","commitTimeStamp":"2016-07-26T09:13:02.8245256Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1771.json","@type":"CatalogPage","commitId":"5b9647d3-2b82-4f14-b1fe-d8d6f0370433","commitTimeStamp":"2016-07-26T17:44:43.9469583Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1772.json","@type":"CatalogPage","commitId":"e64fc4e0-be5e-4286-831b-7c3d537b5a3b","commitTimeStamp":"2016-07-27T08:17:52.1964633Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1773.json","@type":"CatalogPage","commitId":"0794aebf-50fb-4158-8551-da3222a82f26","commitTimeStamp":"2016-07-27T18:25:19.7267169Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1774.json","@type":"CatalogPage","commitId":"8fc38055-c2e5-4e20-be28-3a87ac7afeef","commitTimeStamp":"2016-07-28T09:53:19.9217191Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1775.json","@type":"CatalogPage","commitId":"255eab5c-5fb0-4999-8ad7-b249b6aeb7f2","commitTimeStamp":"2016-07-28T16:01:55.5730703Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1776.json","@type":"CatalogPage","commitId":"c889bf19-2284-416c-a169-63b868d49f5e","commitTimeStamp":"2016-07-29T06:08:47.0992763Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1777.json","@type":"CatalogPage","commitId":"7399aab3-4c44-429f-affb-3d70c20f2796","commitTimeStamp":"2016-07-29T13:31:11.5146341Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page1778.json","@type":"CatalogPage","commitId":"589c87a3-64b8-450d-82ea-cae2ded2f5f3","commitTimeStamp":"2016-07-29T23:39:55.2559989Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1779.json","@type":"CatalogPage","commitId":"c45d8578-4068-44c4-9566-d0f30e22b72d","commitTimeStamp":"2016-07-30T17:58:53.7731995Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1780.json","@type":"CatalogPage","commitId":"9a85ba63-94d7-469e-aa27-7548bf2ec5d5","commitTimeStamp":"2016-07-31T21:13:31.2199278Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1781.json","@type":"CatalogPage","commitId":"1eb0d856-aae6-4a94-ae75-8bf5770f08fd","commitTimeStamp":"2016-08-01T12:17:37.8296177Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1782.json","@type":"CatalogPage","commitId":"89220fab-8fed-42b1-a92c-c477466d86c2","commitTimeStamp":"2016-08-01T21:39:20.5637236Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1783.json","@type":"CatalogPage","commitId":"103b979f-8132-406b-9f37-e20bd1348e65","commitTimeStamp":"2016-08-02T10:26:31.8108976Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1784.json","@type":"CatalogPage","commitId":"3d39ae1b-e629-48f0-88a5-704b76f929aa","commitTimeStamp":"2016-08-02T21:10:35.7534862Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1785.json","@type":"CatalogPage","commitId":"4a15bd7f-96a2-4e57-acd3-898e7b792aa3","commitTimeStamp":"2016-08-03T09:21:24.8178581Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1786.json","@type":"CatalogPage","commitId":"a6cd5cd9-c56c-4b36-89e1-4c9d9a5b6ade","commitTimeStamp":"2016-08-03T18:01:48.4234469Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1787.json","@type":"CatalogPage","commitId":"1909a2da-fb16-4404-b367-bdeef0355d8b","commitTimeStamp":"2016-08-04T06:36:49.9444849Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1788.json","@type":"CatalogPage","commitId":"3e42aab1-8ac0-4c9d-9ce0-ec4da5e6ffb4","commitTimeStamp":"2016-08-04T16:30:11.6838739Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1789.json","@type":"CatalogPage","commitId":"15ebb7bd-28e3-4c18-b691-8a6d9dd6915b","commitTimeStamp":"2016-08-04T23:45:30.3686081Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1790.json","@type":"CatalogPage","commitId":"14e5fb54-febf-4580-9fc1-986b00ed8b94","commitTimeStamp":"2016-08-05T09:32:29.9999799Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1791.json","@type":"CatalogPage","commitId":"b9bf81f5-577c-4b5f-b01d-585ef1815291","commitTimeStamp":"2016-08-05T20:38:15.6792001Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1792.json","@type":"CatalogPage","commitId":"8a0c296b-69b2-4a6e-9a19-1e37ad1f788b","commitTimeStamp":"2016-08-07T02:42:32.9798619Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1793.json","@type":"CatalogPage","commitId":"623d2336-fa9f-42f0-a3e0-c498283de2c6","commitTimeStamp":"2016-08-08T03:51:10.0756687Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1794.json","@type":"CatalogPage","commitId":"a8d77a1a-3780-4040-b2ff-b908d1870d74","commitTimeStamp":"2016-08-08T14:12:57.7600912Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1795.json","@type":"CatalogPage","commitId":"24dd392c-30e9-445b-8934-7c8956620b6d","commitTimeStamp":"2016-08-08T23:03:12.5089488Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1796.json","@type":"CatalogPage","commitId":"18c93ff0-a1a2-4708-a0af-44a5aa3651cf","commitTimeStamp":"2016-08-09T06:14:44.1465332Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1797.json","@type":"CatalogPage","commitId":"f2a37110-282b-4f81-baca-86adf17044c9","commitTimeStamp":"2016-08-09T16:10:10.9021667Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1798.json","@type":"CatalogPage","commitId":"a376ba6c-4912-479a-b8d6-d28a9dd2bb51","commitTimeStamp":"2016-08-09T23:15:26.2382459Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1799.json","@type":"CatalogPage","commitId":"c4107582-e11c-4df2-8f6a-e98671083751","commitTimeStamp":"2016-08-10T10:55:33.5877262Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1800.json","@type":"CatalogPage","commitId":"a02f6a63-2b45-44cf-b104-705dbe7b0e75","commitTimeStamp":"2016-08-10T20:20:07.7121597Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1801.json","@type":"CatalogPage","commitId":"34400ee7-20e6-483c-8632-f094e05e55ba","commitTimeStamp":"2016-08-11T07:15:15.8289476Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1802.json","@type":"CatalogPage","commitId":"a7f3f40e-f031-4245-89b3-189da2c145bb","commitTimeStamp":"2016-08-11T14:02:27.7620521Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1803.json","@type":"CatalogPage","commitId":"264b0a62-4cc3-45f4-acc3-7803c6c69041","commitTimeStamp":"2016-08-11T21:57:26.2814129Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1804.json","@type":"CatalogPage","commitId":"0bdb3783-44ee-4f68-a30f-3192d8d10d4d","commitTimeStamp":"2016-08-12T05:15:48.9540098Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1805.json","@type":"CatalogPage","commitId":"ff5dc45e-3cd2-4385-b3b6-55636ccdf89c","commitTimeStamp":"2016-08-12T15:36:25.7002962Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1806.json","@type":"CatalogPage","commitId":"42c3da50-82c8-4086-9713-9d0eb76542ba","commitTimeStamp":"2016-08-13T07:52:25.5101129Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1807.json","@type":"CatalogPage","commitId":"03c789a4-dc67-47c0-9fe9-674db69f18aa","commitTimeStamp":"2016-08-14T12:23:06.8841966Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1808.json","@type":"CatalogPage","commitId":"9f8f3dbd-c543-4333-b384-e5181c30fcaa","commitTimeStamp":"2016-08-15T10:09:55.793114Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1809.json","@type":"CatalogPage","commitId":"624c8a3c-ec11-449a-8869-928ffc1bc394","commitTimeStamp":"2016-08-15T20:09:58.9307319Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1810.json","@type":"CatalogPage","commitId":"dcf9ec3e-1e83-40bd-b014-0f5ca82b0eb9","commitTimeStamp":"2016-08-16T07:43:14.4914204Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1811.json","@type":"CatalogPage","commitId":"94ec963e-d949-4c5f-aded-0eeba8d4de9d","commitTimeStamp":"2016-08-16T17:33:34.3548606Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1812.json","@type":"CatalogPage","commitId":"bbda8f7e-53a2-4074-b617-106692cf06de","commitTimeStamp":"2016-08-17T09:17:28.3457976Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1813.json","@type":"CatalogPage","commitId":"e6b6e0cb-c0a9-4cb3-bbbc-aec2b64eba4c","commitTimeStamp":"2016-08-17T18:52:46.8982204Z","count":531},{"@id":"https://api.nuget.org/v3/catalog0/page1814.json","@type":"CatalogPage","commitId":"4d783a78-c947-42ff-907a-32b8265b0eb7","commitTimeStamp":"2016-08-18T08:17:32.914784Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1815.json","@type":"CatalogPage","commitId":"84e9a7b0-5a52-4123-8f7f-dcb933801ee2","commitTimeStamp":"2016-08-18T17:50:00.5706066Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1816.json","@type":"CatalogPage","commitId":"322ea699-8bf3-4f9b-949c-febcce8f6aa3","commitTimeStamp":"2016-08-19T05:56:37.934312Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1817.json","@type":"CatalogPage","commitId":"00a71401-4900-4b9c-8368-af89099387d3","commitTimeStamp":"2016-08-19T17:32:20.7072307Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1818.json","@type":"CatalogPage","commitId":"2243c7d6-e40d-4cf6-ac09-b8fc890907a9","commitTimeStamp":"2016-08-20T07:45:14.617952Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1819.json","@type":"CatalogPage","commitId":"36710251-d0dd-4a74-94e9-0cfbf80f870b","commitTimeStamp":"2016-08-21T09:38:20.1659042Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1820.json","@type":"CatalogPage","commitId":"86d42438-05db-4710-9182-5741f6d68a09","commitTimeStamp":"2016-08-22T05:55:52.5496303Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1821.json","@type":"CatalogPage","commitId":"69f85e3d-867a-4f10-99ac-d438ea608262","commitTimeStamp":"2016-08-22T14:05:51.5062219Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1822.json","@type":"CatalogPage","commitId":"cf3bba9d-abfe-4883-8b9f-985de314163e","commitTimeStamp":"2016-08-22T23:25:12.6158356Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1823.json","@type":"CatalogPage","commitId":"162d558d-95df-4f2c-8ed3-a49fcf729a47","commitTimeStamp":"2016-08-23T11:32:52.1001531Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1824.json","@type":"CatalogPage","commitId":"b753e390-204f-4e59-99da-9dd3e4b3127c","commitTimeStamp":"2016-08-23T19:27:36.3955533Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page1825.json","@type":"CatalogPage","commitId":"8405bdd8-87d6-4f03-9bc4-82ca817ec211","commitTimeStamp":"2016-08-24T03:50:59.1253405Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1826.json","@type":"CatalogPage","commitId":"2f0c86bf-0c76-4f21-8ac7-e0309851c79e","commitTimeStamp":"2016-08-24T12:07:05.9431052Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1827.json","@type":"CatalogPage","commitId":"2c4cd519-bc27-49d0-b97d-dc2a84016820","commitTimeStamp":"2016-08-24T21:32:12.5883683Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1828.json","@type":"CatalogPage","commitId":"983eb3c0-fe9d-49aa-bb0d-371ce8e8db73","commitTimeStamp":"2016-08-25T13:10:38.6252074Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1829.json","@type":"CatalogPage","commitId":"ef1b06f4-c703-45ee-b5a0-d2c5947d5606","commitTimeStamp":"2016-08-25T21:02:37.3768006Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1830.json","@type":"CatalogPage","commitId":"bb04d84c-229f-47ea-b02c-78521ef719f1","commitTimeStamp":"2016-08-26T12:04:16.7225472Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1831.json","@type":"CatalogPage","commitId":"7c490ab4-4603-4d0a-84d9-7f93ed652164","commitTimeStamp":"2016-08-27T01:01:18.8763722Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1832.json","@type":"CatalogPage","commitId":"202f8aa1-2bcf-4ae9-bdd6-d48acad79806","commitTimeStamp":"2016-08-28T05:42:41.0672786Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1833.json","@type":"CatalogPage","commitId":"980c0654-26db-47c9-9f7c-35fb5f4b29d9","commitTimeStamp":"2016-08-29T01:18:15.7172776Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1834.json","@type":"CatalogPage","commitId":"8a294540-96a8-44a8-be82-625762abdca9","commitTimeStamp":"2016-08-29T13:08:50.7135569Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1835.json","@type":"CatalogPage","commitId":"836016be-4343-478b-bee4-948d88124df9","commitTimeStamp":"2016-08-29T21:29:57.3267144Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1836.json","@type":"CatalogPage","commitId":"fe673f1b-7b42-4d75-bf8a-a91a4ca2c8cb","commitTimeStamp":"2016-08-30T13:46:58.4935517Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1837.json","@type":"CatalogPage","commitId":"9e877519-b854-486c-a561-1433c9bfb553","commitTimeStamp":"2016-08-31T03:42:38.4886456Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1838.json","@type":"CatalogPage","commitId":"5da525d1-1466-49ce-b79e-f94b9533613c","commitTimeStamp":"2016-08-31T15:05:34.226642Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1839.json","@type":"CatalogPage","commitId":"c6cefed4-ea81-4ef9-8908-f945847b1a33","commitTimeStamp":"2016-09-01T00:39:10.4178926Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1840.json","@type":"CatalogPage","commitId":"cd8f8dd5-7409-478c-b442-9698f53b6ea6","commitTimeStamp":"2016-09-01T14:01:33.2725832Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1841.json","@type":"CatalogPage","commitId":"c1dc18e0-d48d-4fd0-b787-2914c51ea2f1","commitTimeStamp":"2016-09-01T21:53:45.2948304Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1842.json","@type":"CatalogPage","commitId":"c8e4da9b-3cab-4262-b763-abe044a7677d","commitTimeStamp":"2016-09-02T09:34:30.4316766Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1843.json","@type":"CatalogPage","commitId":"c65b5e62-2429-42eb-8a01-77e51033c3b1","commitTimeStamp":"2016-09-02T16:00:17.6014451Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1844.json","@type":"CatalogPage","commitId":"64001c7a-f9fa-4460-8bcc-bdcf7cbd1bf8","commitTimeStamp":"2016-09-03T12:36:07.9662686Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1845.json","@type":"CatalogPage","commitId":"79a18e1f-67f0-43c8-9a6f-504082d112ee","commitTimeStamp":"2016-09-04T09:39:32.2558583Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1846.json","@type":"CatalogPage","commitId":"85fa00db-39a0-43c9-bc4a-c06c18292795","commitTimeStamp":"2016-09-05T04:50:55.6276597Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1847.json","@type":"CatalogPage","commitId":"1a6ab1ed-abff-4828-b653-1385bcb81af2","commitTimeStamp":"2016-09-05T14:16:23.9510327Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page1848.json","@type":"CatalogPage","commitId":"44777791-e990-4983-b548-bb41559cacc8","commitTimeStamp":"2016-09-06T08:14:16.2885642Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1849.json","@type":"CatalogPage","commitId":"6f7be2ba-b9ab-42a3-8ff4-9547a3ad508e","commitTimeStamp":"2016-09-06T16:03:00.0443883Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1850.json","@type":"CatalogPage","commitId":"8c0e5e6d-984e-46ed-bcd8-68e067fa57e4","commitTimeStamp":"2016-09-07T03:21:18.2343711Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1851.json","@type":"CatalogPage","commitId":"ce4a6071-3a31-4258-91f5-0b869b57251b","commitTimeStamp":"2016-09-07T11:53:00.2779647Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1852.json","@type":"CatalogPage","commitId":"0b2f4640-eee9-4212-bc16-e555fbb944eb","commitTimeStamp":"2016-09-07T20:42:18.1662823Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1853.json","@type":"CatalogPage","commitId":"211c6b7f-5169-4787-a9e4-53cbab3fb066","commitTimeStamp":"2016-09-08T09:21:06.3864017Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1854.json","@type":"CatalogPage","commitId":"f160443d-2fed-4b22-a875-e80f91be006f","commitTimeStamp":"2016-09-08T14:18:47.8513311Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1855.json","@type":"CatalogPage","commitId":"2aac359f-ca73-4169-9327-917403a248ae","commitTimeStamp":"2016-09-09T00:32:31.3002268Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1856.json","@type":"CatalogPage","commitId":"c25c69f1-18b8-4439-96df-b8f93f45202e","commitTimeStamp":"2016-09-09T12:13:14.0182822Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1857.json","@type":"CatalogPage","commitId":"8b919a6c-123c-451d-b59d-1baf1ce13723","commitTimeStamp":"2016-09-09T21:24:49.1869994Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page1858.json","@type":"CatalogPage","commitId":"eabac624-d6b0-454f-8b1a-769805dae12c","commitTimeStamp":"2016-09-10T14:53:31.613225Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1859.json","@type":"CatalogPage","commitId":"228dd752-c0fa-4fe0-9d85-1e946d6180d1","commitTimeStamp":"2016-09-11T16:26:07.6408215Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1860.json","@type":"CatalogPage","commitId":"0ba184a9-c191-45db-aa1c-108eb989a5c8","commitTimeStamp":"2016-09-12T06:34:35.7666298Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1861.json","@type":"CatalogPage","commitId":"eff22441-62d4-4c49-9350-08d66c83e8b2","commitTimeStamp":"2016-09-12T11:32:44.0569107Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1862.json","@type":"CatalogPage","commitId":"7c2b7f15-4284-4da2-a7bc-268f175347f1","commitTimeStamp":"2016-09-12T17:17:44.9523964Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1863.json","@type":"CatalogPage","commitId":"f37cc05a-30cf-4a26-b400-dab005b12fed","commitTimeStamp":"2016-09-13T09:48:23.4977461Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1864.json","@type":"CatalogPage","commitId":"aeeb098e-a869-480b-be91-79870b2df043","commitTimeStamp":"2016-09-13T18:51:05.8408126Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1865.json","@type":"CatalogPage","commitId":"dda6eb65-1af6-45a3-92c9-2bf076829351","commitTimeStamp":"2016-09-14T08:11:51.759113Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1866.json","@type":"CatalogPage","commitId":"7ab3fbc8-eee4-4fdf-979a-fc95fbff6abc","commitTimeStamp":"2016-09-14T15:50:14.7866253Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1867.json","@type":"CatalogPage","commitId":"9e1d1dce-be6f-4f7d-9a70-66bfe9d7c8c5","commitTimeStamp":"2016-09-15T08:14:44.5603852Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1868.json","@type":"CatalogPage","commitId":"60c6fef6-2c80-44e9-91d2-323ee071c761","commitTimeStamp":"2016-09-15T16:35:27.1266557Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1869.json","@type":"CatalogPage","commitId":"06ea7112-e2aa-4a56-8952-6f2b8f6c44fe","commitTimeStamp":"2016-09-16T04:14:59.9404569Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1870.json","@type":"CatalogPage","commitId":"edfeb9c0-161b-49ec-b39f-b295d14d4053","commitTimeStamp":"2016-09-16T15:38:26.3034519Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1871.json","@type":"CatalogPage","commitId":"4095bdb4-996f-47cc-8f1d-39b7c9d26c65","commitTimeStamp":"2016-09-17T08:46:45.5122926Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1872.json","@type":"CatalogPage","commitId":"ac18574a-1794-40e5-849b-8b4805c9a9b6","commitTimeStamp":"2016-09-18T10:58:15.116096Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1873.json","@type":"CatalogPage","commitId":"dd806565-f373-45e2-ad0f-21810e0e7124","commitTimeStamp":"2016-09-19T08:12:40.5157532Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1874.json","@type":"CatalogPage","commitId":"f784482c-d0bb-4f87-8854-85e48ae5a6a2","commitTimeStamp":"2016-09-19T16:18:28.9371105Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page1875.json","@type":"CatalogPage","commitId":"d4d19151-35c9-48c0-bf37-7454fa860b63","commitTimeStamp":"2016-09-20T04:41:06.2627161Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1876.json","@type":"CatalogPage","commitId":"19792637-4325-4ba5-85a5-3eb135fb2607","commitTimeStamp":"2016-09-20T15:41:24.8207281Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1877.json","@type":"CatalogPage","commitId":"3a8ca0fc-70db-4a60-8bbd-65e79947ea71","commitTimeStamp":"2016-09-21T02:39:23.1443329Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1878.json","@type":"CatalogPage","commitId":"a1c7dce3-57fa-45de-bdb5-8a18ad40314f","commitTimeStamp":"2016-09-21T14:42:52.036902Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1879.json","@type":"CatalogPage","commitId":"eb6b00dd-68cc-4539-8c66-feec1aae537a","commitTimeStamp":"2016-09-22T02:37:01.0126116Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1880.json","@type":"CatalogPage","commitId":"ae162836-c38d-44e8-8992-20cb5efe7135","commitTimeStamp":"2016-09-22T15:50:01.4817788Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1881.json","@type":"CatalogPage","commitId":"4a69c46f-5447-4bac-a2c0-8f9d47ef9915","commitTimeStamp":"2016-09-23T02:27:50.9081441Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1882.json","@type":"CatalogPage","commitId":"4568ee08-ae8a-4e22-b783-54156cc5f302","commitTimeStamp":"2016-09-23T13:58:01.196172Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1883.json","@type":"CatalogPage","commitId":"b5a0e755-c2e8-43da-831a-69a7de2d6316","commitTimeStamp":"2016-09-24T07:29:28.963043Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1884.json","@type":"CatalogPage","commitId":"5eae059c-6a86-476e-914d-795e8202c11b","commitTimeStamp":"2016-09-25T10:06:18.7779008Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1885.json","@type":"CatalogPage","commitId":"4ff66548-4b5d-45f4-ae8f-e426f29d8f09","commitTimeStamp":"2016-09-26T04:19:22.9743953Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1886.json","@type":"CatalogPage","commitId":"94e26d42-ea31-492b-8232-ef4a424287db","commitTimeStamp":"2016-09-26T14:33:46.0457709Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1887.json","@type":"CatalogPage","commitId":"4a00db8a-d125-4e29-b80f-0dfdad9300a8","commitTimeStamp":"2016-09-27T02:48:03.782371Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1888.json","@type":"CatalogPage","commitId":"6a2633ca-76b6-4a9c-b0d7-6cf733732ddc","commitTimeStamp":"2016-09-27T12:51:50.4752378Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1889.json","@type":"CatalogPage","commitId":"edab1e55-e060-4431-8cc0-9159058c9512","commitTimeStamp":"2016-09-27T20:30:17.4841074Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1890.json","@type":"CatalogPage","commitId":"a00e51a0-516c-4365-99b3-d50bc5ec2fdd","commitTimeStamp":"2016-09-28T08:09:53.7814111Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1891.json","@type":"CatalogPage","commitId":"99552a2f-19b3-4d7d-8d0d-cbfeb72ea379","commitTimeStamp":"2016-09-28T16:50:22.9911976Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1892.json","@type":"CatalogPage","commitId":"7283b21c-84f8-48cc-bd1e-4eacab1949fe","commitTimeStamp":"2016-09-28T23:53:53.7102915Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1893.json","@type":"CatalogPage","commitId":"b5ab6c30-03e1-474f-8ed2-70b92ce58fb7","commitTimeStamp":"2016-09-29T09:39:56.3069594Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1894.json","@type":"CatalogPage","commitId":"435d6b24-9cbc-437a-8e83-9fff4810ab96","commitTimeStamp":"2016-09-29T16:05:47.3304017Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1895.json","@type":"CatalogPage","commitId":"09ab3527-d1d6-42f2-bf93-7d58508e1243","commitTimeStamp":"2016-09-30T07:21:45.755795Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1896.json","@type":"CatalogPage","commitId":"31c81ffc-ac21-45c4-83c9-6f8298239c38","commitTimeStamp":"2016-09-30T15:54:01.3443168Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1897.json","@type":"CatalogPage","commitId":"6a0b6f63-f602-4176-9a07-eef66f14deb7","commitTimeStamp":"2016-10-01T02:03:23.0259315Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1898.json","@type":"CatalogPage","commitId":"6caecf5b-b502-456d-93bd-0b258675f6b9","commitTimeStamp":"2016-10-02T11:43:57.0784488Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1899.json","@type":"CatalogPage","commitId":"7aa3f5b0-a24a-4a50-9d32-8cd4a2f16cb9","commitTimeStamp":"2016-10-03T08:06:10.7908586Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1900.json","@type":"CatalogPage","commitId":"3e3224e0-5c3e-44ae-955e-9605d6e46481","commitTimeStamp":"2016-10-03T16:00:36.4101923Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1901.json","@type":"CatalogPage","commitId":"79fcde65-1a6a-48b2-a7ed-8e7c613d76c2","commitTimeStamp":"2016-10-03T21:10:14.9340189Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page1902.json","@type":"CatalogPage","commitId":"b64e5ac2-0371-4ac4-9678-c45c76d62ec8","commitTimeStamp":"2016-10-04T09:30:30.1865303Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1903.json","@type":"CatalogPage","commitId":"abf1c950-3377-4c7e-b353-5f0bf6d96082","commitTimeStamp":"2016-10-04T19:17:48.2814248Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1904.json","@type":"CatalogPage","commitId":"a5a71622-c55e-47ba-aa3f-14ac2f254189","commitTimeStamp":"2016-10-05T13:25:45.6444531Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1905.json","@type":"CatalogPage","commitId":"d9f32afa-8707-4c9b-a7bd-745131c2c38a","commitTimeStamp":"2016-10-05T21:31:56.7392617Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1906.json","@type":"CatalogPage","commitId":"de42e899-2e33-4fe5-b5db-ea6f5afeb0fe","commitTimeStamp":"2016-10-06T09:52:45.2409383Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1907.json","@type":"CatalogPage","commitId":"4ee8c6ca-0c7c-4179-85c1-027fb4ec0fb0","commitTimeStamp":"2016-10-06T16:35:48.9660025Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1908.json","@type":"CatalogPage","commitId":"5480a9dc-b21f-40ae-8156-4e5751eda374","commitTimeStamp":"2016-10-06T21:58:52.391887Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1909.json","@type":"CatalogPage","commitId":"61b0ac82-5475-48c1-bd1d-69a35d2254fd","commitTimeStamp":"2016-10-07T09:41:46.7398379Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1910.json","@type":"CatalogPage","commitId":"8431314b-e3ef-4848-be86-579ac266e001","commitTimeStamp":"2016-10-07T19:23:59.3619408Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1911.json","@type":"CatalogPage","commitId":"1d598008-23d6-4003-973b-182b35c5b7ed","commitTimeStamp":"2016-10-08T15:15:39.0713005Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1912.json","@type":"CatalogPage","commitId":"fe9fe94d-f408-4f35-bf8f-5476ef4b7254","commitTimeStamp":"2016-10-09T16:57:08.1043599Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1913.json","@type":"CatalogPage","commitId":"a0f5a580-35e8-40ed-9bc1-6bfc939fc0d2","commitTimeStamp":"2016-10-10T08:16:51.2257602Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1914.json","@type":"CatalogPage","commitId":"7682320f-c238-420b-bede-80222074d7b0","commitTimeStamp":"2016-10-10T16:06:18.4911469Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1915.json","@type":"CatalogPage","commitId":"1615248c-250c-4510-9517-87544b7c12bf","commitTimeStamp":"2016-10-11T06:46:45.1573847Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1916.json","@type":"CatalogPage","commitId":"61367763-5f4d-4105-b941-d3e32aac6af6","commitTimeStamp":"2016-10-11T14:52:21.9114905Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1917.json","@type":"CatalogPage","commitId":"b4a09844-cad6-466a-a6ed-14c57e362d35","commitTimeStamp":"2016-10-12T01:34:12.4824143Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1918.json","@type":"CatalogPage","commitId":"95377ca0-c373-4ecf-b2b7-3a55c0f64e2f","commitTimeStamp":"2016-10-12T13:30:57.9601348Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1919.json","@type":"CatalogPage","commitId":"ba6d4e51-e51f-471a-8ec6-41a0242681ca","commitTimeStamp":"2016-10-12T22:13:13.1143665Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1920.json","@type":"CatalogPage","commitId":"2ba15871-89dd-4380-9ffb-c16fcf474765","commitTimeStamp":"2016-10-13T06:46:08.6421335Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1921.json","@type":"CatalogPage","commitId":"11a82196-3981-4989-9d8b-26c587bc27cc","commitTimeStamp":"2016-10-13T13:37:55.6770333Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1922.json","@type":"CatalogPage","commitId":"8ad6e3e9-7ec8-4cdc-af2c-27e24b080daa","commitTimeStamp":"2016-10-13T21:25:25.5035315Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1923.json","@type":"CatalogPage","commitId":"fee09fba-4c1e-4f71-86e5-ff71380eb469","commitTimeStamp":"2016-10-14T09:33:29.656747Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1924.json","@type":"CatalogPage","commitId":"d7a053ba-20ab-4964-ba6e-9b08f4ec37fe","commitTimeStamp":"2016-10-14T20:54:56.2056384Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1925.json","@type":"CatalogPage","commitId":"5a0ce8ce-0275-4c9d-9521-b8be99384d46","commitTimeStamp":"2016-10-15T16:19:47.352807Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1926.json","@type":"CatalogPage","commitId":"341f097b-c60d-4f0e-a65e-0b8cbe18509d","commitTimeStamp":"2016-10-16T12:52:03.196116Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1927.json","@type":"CatalogPage","commitId":"083bbca3-803e-4937-808c-8185b48efc21","commitTimeStamp":"2016-10-17T03:38:34.3093716Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1928.json","@type":"CatalogPage","commitId":"8d5160ab-9986-4f23-9f77-1256b108ef65","commitTimeStamp":"2016-10-17T12:39:20.1542642Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1929.json","@type":"CatalogPage","commitId":"f85f5bb0-5b03-4515-be69-4743f737226d","commitTimeStamp":"2016-10-17T22:40:00.9324523Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1930.json","@type":"CatalogPage","commitId":"a811d1e9-5d37-494d-a412-fed2971ce226","commitTimeStamp":"2016-10-18T09:01:43.0404462Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1931.json","@type":"CatalogPage","commitId":"30427605-1082-4413-adbf-4fc39bf7d63e","commitTimeStamp":"2016-10-18T18:40:38.1919417Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1932.json","@type":"CatalogPage","commitId":"6eb78617-96cd-490b-9461-d2c9ccfe4b1d","commitTimeStamp":"2016-10-19T08:21:17.6468615Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1933.json","@type":"CatalogPage","commitId":"a8146e22-6a9e-4717-808a-d80933e956bb","commitTimeStamp":"2016-10-19T15:59:55.0859137Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1934.json","@type":"CatalogPage","commitId":"6e760176-b268-4a64-837e-0a26fd2e7398","commitTimeStamp":"2016-10-19T23:15:05.9666421Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1935.json","@type":"CatalogPage","commitId":"75d8b86c-10cf-4985-9735-b1d57294ea25","commitTimeStamp":"2016-10-20T09:39:52.4930736Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1936.json","@type":"CatalogPage","commitId":"bea68b12-021e-41e6-a767-e9926abb29c2","commitTimeStamp":"2016-10-20T15:37:19.6144935Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1937.json","@type":"CatalogPage","commitId":"fd782d98-096a-48a4-8ae1-074c51fb2a6b","commitTimeStamp":"2016-10-20T21:58:35.1847325Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1938.json","@type":"CatalogPage","commitId":"53a9246d-b0bd-4d23-bfa8-1e277e0e4769","commitTimeStamp":"2016-10-21T11:31:22.8927907Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page1939.json","@type":"CatalogPage","commitId":"e98a4bcb-1c53-46d5-b347-788084262421","commitTimeStamp":"2016-10-21T20:28:42.7024538Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1940.json","@type":"CatalogPage","commitId":"14783c3f-64c3-4b38-a6c7-27b43e50d467","commitTimeStamp":"2016-10-22T17:59:57.4975679Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1941.json","@type":"CatalogPage","commitId":"07869d9c-c031-4399-be90-7ad000130461","commitTimeStamp":"2016-10-23T14:50:23.2852496Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1942.json","@type":"CatalogPage","commitId":"90597ce8-ab5e-4603-8035-1f53081cdee4","commitTimeStamp":"2016-10-24T09:50:34.6924784Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1943.json","@type":"CatalogPage","commitId":"5206e3ac-c30d-4fe7-8b31-ac1a5efad7a0","commitTimeStamp":"2016-10-24T16:12:28.2381162Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1944.json","@type":"CatalogPage","commitId":"a163f3f8-7335-4588-bd61-cd07bb99b4b3","commitTimeStamp":"2016-10-24T17:40:39.3219838Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1945.json","@type":"CatalogPage","commitId":"dcce0c0a-dc48-4508-8848-c5189cfee19d","commitTimeStamp":"2016-10-25T01:44:43.2244753Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1946.json","@type":"CatalogPage","commitId":"a320ebd1-4bef-432c-8a8a-5a672064b497","commitTimeStamp":"2016-10-25T09:47:04.3793176Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1947.json","@type":"CatalogPage","commitId":"bebfafe2-ddd1-4602-83c6-ffc4efb333df","commitTimeStamp":"2016-10-25T17:09:12.6407067Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1948.json","@type":"CatalogPage","commitId":"62285fbc-fdb4-41eb-865e-a6b0c76b6d09","commitTimeStamp":"2016-10-25T23:49:43.8409195Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1949.json","@type":"CatalogPage","commitId":"70af40db-69f4-42f0-91ac-e43cc88945cc","commitTimeStamp":"2016-10-26T10:03:36.2029226Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1950.json","@type":"CatalogPage","commitId":"f544c5f2-bfba-436b-a6ed-b2877d74db85","commitTimeStamp":"2016-10-26T16:29:26.4819328Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1951.json","@type":"CatalogPage","commitId":"430acaae-8e4a-4976-ab0f-8ae290692d36","commitTimeStamp":"2016-10-27T02:40:52.2376228Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1952.json","@type":"CatalogPage","commitId":"177ff4f0-9661-46d2-8c67-6af14477ca8a","commitTimeStamp":"2016-10-27T13:37:30.8641475Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1953.json","@type":"CatalogPage","commitId":"86d9bfbf-26cf-476f-88e5-f4597660f0fc","commitTimeStamp":"2016-10-27T20:01:55.824019Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1954.json","@type":"CatalogPage","commitId":"959df6d3-a6be-4780-9565-482ed546073a","commitTimeStamp":"2016-10-28T06:14:39.3850818Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1955.json","@type":"CatalogPage","commitId":"dc397de8-508a-4b7f-a7eb-e434d9dbb028","commitTimeStamp":"2016-10-28T15:33:33.6723406Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1956.json","@type":"CatalogPage","commitId":"01f256f7-7a34-49ef-bca1-d59cecbbac1f","commitTimeStamp":"2016-10-29T06:01:48.6411878Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1957.json","@type":"CatalogPage","commitId":"139196e9-7ac6-4b08-85b3-330ecf864ebc","commitTimeStamp":"2016-10-30T07:01:25.4390725Z","count":537},{"@id":"https://api.nuget.org/v3/catalog0/page1958.json","@type":"CatalogPage","commitId":"8b346d4c-84b3-4d86-b53e-edd99d32baa5","commitTimeStamp":"2016-10-30T18:11:40.1236863Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1959.json","@type":"CatalogPage","commitId":"a94ba521-7163-4bda-b2bd-692577c527ec","commitTimeStamp":"2016-10-31T08:32:09.3594476Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1960.json","@type":"CatalogPage","commitId":"227e1c80-d090-44ac-afca-b98cb53aa648","commitTimeStamp":"2016-10-31T16:45:43.7945537Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1961.json","@type":"CatalogPage","commitId":"c5a3b7be-66f5-4470-b2b7-a099f1c60f78","commitTimeStamp":"2016-11-01T05:33:57.3578841Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1962.json","@type":"CatalogPage","commitId":"04296478-3e06-478b-b691-29adfe8bfc1c","commitTimeStamp":"2016-11-01T11:05:39.8540235Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1963.json","@type":"CatalogPage","commitId":"f83e08cb-c497-4611-87e1-c104b38ee7fb","commitTimeStamp":"2016-11-01T17:35:58.3921919Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1964.json","@type":"CatalogPage","commitId":"5507fe70-a231-4cfe-b48b-2bb57ac4194e","commitTimeStamp":"2016-11-02T02:18:04.8560993Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1965.json","@type":"CatalogPage","commitId":"049a1582-7818-49ac-bc8a-8bc109dff45b","commitTimeStamp":"2016-11-02T13:46:19.175897Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1966.json","@type":"CatalogPage","commitId":"687d25eb-24ea-4cb4-9857-d269cb31280d","commitTimeStamp":"2016-11-02T21:20:11.9566991Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1967.json","@type":"CatalogPage","commitId":"3638da71-af4e-443d-8bab-04c41af457f9","commitTimeStamp":"2016-11-03T11:17:19.788194Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1968.json","@type":"CatalogPage","commitId":"770a9eec-156e-4ac6-bd57-7b0463acd5d9","commitTimeStamp":"2016-11-03T18:50:01.5586634Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1969.json","@type":"CatalogPage","commitId":"1949b75f-36fa-4a04-bebe-a9bfc590feac","commitTimeStamp":"2016-11-04T08:34:26.4986043Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1970.json","@type":"CatalogPage","commitId":"931a9e57-217c-41bd-9a55-026a6799b2db","commitTimeStamp":"2016-11-04T16:03:27.3913804Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1971.json","@type":"CatalogPage","commitId":"22890c24-0a81-4a95-9593-a337509602fd","commitTimeStamp":"2016-11-05T04:42:52.3694437Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1972.json","@type":"CatalogPage","commitId":"c269e022-57c1-4e7b-97c5-26e02d6055da","commitTimeStamp":"2016-11-05T22:38:32.3947059Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1973.json","@type":"CatalogPage","commitId":"a8877821-eba1-4d28-95f9-8111c8d8a202","commitTimeStamp":"2016-11-07T02:22:09.2825509Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1974.json","@type":"CatalogPage","commitId":"f9d4718a-1a53-4bbe-9ddb-f3142f7d33bd","commitTimeStamp":"2016-11-07T12:41:16.3981256Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1975.json","@type":"CatalogPage","commitId":"3a088cab-9c8b-4672-9b7f-a2f2e54a18d3","commitTimeStamp":"2016-11-07T21:35:22.488288Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1976.json","@type":"CatalogPage","commitId":"a74a4e31-8a86-4e28-b41d-14167d7db5da","commitTimeStamp":"2016-11-08T08:25:52.4572979Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1977.json","@type":"CatalogPage","commitId":"6cdab327-a0ba-4e0d-aaa8-79cb3ae72eb8","commitTimeStamp":"2016-11-08T13:24:02.6783584Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1978.json","@type":"CatalogPage","commitId":"9d02e08d-6def-4f28-9bd3-27dcb4ee55ee","commitTimeStamp":"2016-11-09T02:26:29.026772Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1979.json","@type":"CatalogPage","commitId":"99673553-5d7f-4931-9752-2341bb3b7fcd","commitTimeStamp":"2016-11-09T13:44:54.9505872Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1980.json","@type":"CatalogPage","commitId":"b36a80e1-995e-40f1-917d-60457e0a0b0e","commitTimeStamp":"2016-11-09T22:25:54.4939615Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1981.json","@type":"CatalogPage","commitId":"68b7dd43-6e57-42b4-a8b7-4f0d0ff86043","commitTimeStamp":"2016-11-10T12:08:07.3365122Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1982.json","@type":"CatalogPage","commitId":"ec92034b-ea19-4c30-86fc-3b3cb3687b09","commitTimeStamp":"2016-11-10T18:56:30.7374922Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1983.json","@type":"CatalogPage","commitId":"dc0aeac6-4d7c-4480-bb68-f329ca023e7b","commitTimeStamp":"2016-11-10T22:01:17.9517274Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page1984.json","@type":"CatalogPage","commitId":"fd27333e-95aa-4dd7-b999-bbbe1f625507","commitTimeStamp":"2016-11-11T09:42:03.1437369Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1985.json","@type":"CatalogPage","commitId":"90d18a78-ba74-4988-b276-98e4af6df91e","commitTimeStamp":"2016-11-11T18:11:57.8031879Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1986.json","@type":"CatalogPage","commitId":"babce24d-535b-4289-9db2-e014e5706f5f","commitTimeStamp":"2016-11-12T05:23:24.9812042Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1987.json","@type":"CatalogPage","commitId":"77fd4a50-177d-4857-a089-1d93148d3f2e","commitTimeStamp":"2016-11-12T23:40:37.670238Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1988.json","@type":"CatalogPage","commitId":"71f9d6ed-3399-4cd3-8e73-da0f24f65701","commitTimeStamp":"2016-11-13T21:09:32.9805705Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1989.json","@type":"CatalogPage","commitId":"a2f6f007-9c92-48c1-a82f-8e4fc811941a","commitTimeStamp":"2016-11-14T11:26:08.4908195Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1990.json","@type":"CatalogPage","commitId":"f4ec6dc1-ef54-4e90-bc77-b83bcd496ae3","commitTimeStamp":"2016-11-14T21:56:20.5932048Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1991.json","@type":"CatalogPage","commitId":"014de31c-fc7c-4359-b81a-5fe8daade19c","commitTimeStamp":"2016-11-15T10:10:57.1979489Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1992.json","@type":"CatalogPage","commitId":"d71f69da-ba80-4b3a-9e43-c03ee33d93f0","commitTimeStamp":"2016-11-15T18:21:17.3018488Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page1993.json","@type":"CatalogPage","commitId":"3088ba5c-d43a-4c4d-9e70-68a4aa46c8b6","commitTimeStamp":"2016-11-15T23:35:35.2988913Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1994.json","@type":"CatalogPage","commitId":"d757f295-abe3-4ef0-9e69-56254f907dab","commitTimeStamp":"2016-11-16T03:50:07.4505881Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page1995.json","@type":"CatalogPage","commitId":"6090a2d9-99e6-49bc-8aad-343de591d5ab","commitTimeStamp":"2016-11-16T11:32:38.3432421Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1996.json","@type":"CatalogPage","commitId":"a4e559d9-26d4-4f8e-9261-fd9a23e5de7b","commitTimeStamp":"2016-11-16T18:14:48.1862278Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1997.json","@type":"CatalogPage","commitId":"773a893d-2fdf-4080-b133-32b06ea9c81f","commitTimeStamp":"2016-11-16T22:48:38.4620748Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1998.json","@type":"CatalogPage","commitId":"916a3dcb-5014-4564-bd2f-2d0a28b02c15","commitTimeStamp":"2016-11-17T06:53:07.7877697Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page1999.json","@type":"CatalogPage","commitId":"83aa09f4-9ca3-42db-9326-7da120f0b7ca","commitTimeStamp":"2016-11-17T14:27:12.4050545Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2000.json","@type":"CatalogPage","commitId":"07a6d672-aeb8-43a8-a554-f8d330fa3f44","commitTimeStamp":"2016-11-17T19:21:18.4623941Z","count":536},{"@id":"https://api.nuget.org/v3/catalog0/page2001.json","@type":"CatalogPage","commitId":"4ef04d0a-e9ef-4af7-9021-d305134fe4ea","commitTimeStamp":"2016-11-18T03:19:18.4385884Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2002.json","@type":"CatalogPage","commitId":"b7922980-f400-47d9-b3e8-43a362b4e11c","commitTimeStamp":"2016-11-18T13:10:12.4968949Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2003.json","@type":"CatalogPage","commitId":"56270959-7d47-4e3a-b203-35cb0c67c028","commitTimeStamp":"2016-11-18T19:46:18.8847925Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2004.json","@type":"CatalogPage","commitId":"9e918e74-563c-4183-8705-5f2372a4d1b1","commitTimeStamp":"2016-11-18T21:09:14.7142855Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2005.json","@type":"CatalogPage","commitId":"a925bc4e-5622-4c16-ad2a-6acae60af9e3","commitTimeStamp":"2016-11-19T13:22:28.0833891Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2006.json","@type":"CatalogPage","commitId":"930294d3-a992-49e8-b51a-c8f6c2e29425","commitTimeStamp":"2016-11-20T09:14:22.6644997Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2007.json","@type":"CatalogPage","commitId":"c2e37fd7-8636-462e-80e5-2bc38b7f05c0","commitTimeStamp":"2016-11-21T05:17:56.5115088Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2008.json","@type":"CatalogPage","commitId":"a372e8e5-4128-4460-8a30-96fb1a9d2be7","commitTimeStamp":"2016-11-21T09:58:06.7218924Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2009.json","@type":"CatalogPage","commitId":"f647e6e4-9b61-462a-ac8a-40dbee8e5fea","commitTimeStamp":"2016-11-21T15:21:28.0343053Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2010.json","@type":"CatalogPage","commitId":"4b97dde4-fc23-4477-a346-8b5cff14ca15","commitTimeStamp":"2016-11-21T21:28:27.0720383Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2011.json","@type":"CatalogPage","commitId":"558c27f0-24c2-4ffe-8cac-c2ac80d5c2c9","commitTimeStamp":"2016-11-22T01:16:17.9025289Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2012.json","@type":"CatalogPage","commitId":"12b52697-9ce6-4a88-bb44-85fb82800b3e","commitTimeStamp":"2016-11-22T11:40:19.4200436Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2013.json","@type":"CatalogPage","commitId":"e74755c9-ba34-484f-9d1b-c9224e461346","commitTimeStamp":"2016-11-22T18:36:13.9705018Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2014.json","@type":"CatalogPage","commitId":"f8cfd8b1-ae32-48d4-82aa-82a38fdba672","commitTimeStamp":"2016-11-23T05:08:35.6850346Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2015.json","@type":"CatalogPage","commitId":"6a8ed392-28cb-44b4-80eb-6c2378ebdef2","commitTimeStamp":"2016-11-23T13:28:03.6665459Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2016.json","@type":"CatalogPage","commitId":"5a4f64c3-ed20-4f35-b184-79d42a861512","commitTimeStamp":"2016-11-23T22:14:51.8236276Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2017.json","@type":"CatalogPage","commitId":"b12d3caa-4fa8-46a7-82ee-4359553aea02","commitTimeStamp":"2016-11-24T10:12:25.9393916Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2018.json","@type":"CatalogPage","commitId":"ecd8229e-621a-4e0a-b4e4-087d0b345a66","commitTimeStamp":"2016-11-24T17:33:43.7508662Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2019.json","@type":"CatalogPage","commitId":"5c3392fa-52d8-4504-a0b2-73040d7eb1ab","commitTimeStamp":"2016-11-25T09:04:32.0461901Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2020.json","@type":"CatalogPage","commitId":"e5ba96b8-e363-48fb-9805-feaeffd06f90","commitTimeStamp":"2016-11-25T15:05:38.0600803Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2021.json","@type":"CatalogPage","commitId":"473d8bb1-224c-44da-89e3-7c619d2f1554","commitTimeStamp":"2016-11-26T16:36:15.3250759Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2022.json","@type":"CatalogPage","commitId":"d21d8ef8-b685-4450-aa22-b440238e0e0e","commitTimeStamp":"2016-11-27T17:26:59.2526048Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2023.json","@type":"CatalogPage","commitId":"c422152d-ab93-447c-bd9e-55163370ea47","commitTimeStamp":"2016-11-28T09:43:17.677544Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2024.json","@type":"CatalogPage","commitId":"8519a291-666e-4cdf-87bb-5cfde72afe66","commitTimeStamp":"2016-11-28T20:13:26.2102115Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2025.json","@type":"CatalogPage","commitId":"45f7c7be-c84e-4373-91df-35058482b125","commitTimeStamp":"2016-11-29T09:41:20.2166023Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2026.json","@type":"CatalogPage","commitId":"f058bb98-ff37-4d80-89ba-984649cb76f4","commitTimeStamp":"2016-11-29T17:39:27.1428373Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2027.json","@type":"CatalogPage","commitId":"33ea7743-efc9-4595-90a4-bd4ae3ff7973","commitTimeStamp":"2016-11-30T01:51:16.4248937Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2028.json","@type":"CatalogPage","commitId":"bf41bd8a-7296-4767-afe1-c6e01d560689","commitTimeStamp":"2016-11-30T09:39:50.6390912Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2029.json","@type":"CatalogPage","commitId":"49b3040d-14f9-4775-af2f-4c259c004cc0","commitTimeStamp":"2016-11-30T18:10:24.0780735Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2030.json","@type":"CatalogPage","commitId":"97365930-8d64-417c-a528-d8046e3f3a78","commitTimeStamp":"2016-12-01T05:54:40.7568164Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2031.json","@type":"CatalogPage","commitId":"510f4f35-f7b0-4232-8774-00f190651fad","commitTimeStamp":"2016-12-01T17:21:33.3629633Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2032.json","@type":"CatalogPage","commitId":"ca98f799-ecc6-4d60-b116-00fb12455a70","commitTimeStamp":"2016-12-02T01:44:38.7278365Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2033.json","@type":"CatalogPage","commitId":"247badb2-f6bd-44f8-bb46-1595b933a838","commitTimeStamp":"2016-12-02T15:35:22.5584731Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2034.json","@type":"CatalogPage","commitId":"92c755fc-2993-4491-a50c-fb99044dc8a1","commitTimeStamp":"2016-12-03T08:52:05.9964698Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2035.json","@type":"CatalogPage","commitId":"f1b28aac-d435-45b4-9857-cba78a170b46","commitTimeStamp":"2016-12-03T18:14:05.9900222Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2036.json","@type":"CatalogPage","commitId":"ec266521-c461-418d-84a0-b5aa520f48b0","commitTimeStamp":"2016-12-04T12:27:47.1969237Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2037.json","@type":"CatalogPage","commitId":"9aea334d-d78a-4066-b6c2-519782f6aa05","commitTimeStamp":"2016-12-05T05:01:35.1300141Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2038.json","@type":"CatalogPage","commitId":"e24729d5-5263-483e-97be-52d542416ae7","commitTimeStamp":"2016-12-05T13:58:57.5767415Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2039.json","@type":"CatalogPage","commitId":"7ec040c5-a3a9-4463-a548-48202c06db03","commitTimeStamp":"2016-12-05T19:42:29.1108607Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2040.json","@type":"CatalogPage","commitId":"0f2477ca-f6d7-468b-a4b3-7a49c21ffbe6","commitTimeStamp":"2016-12-06T08:15:46.5902668Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2041.json","@type":"CatalogPage","commitId":"cdd714b6-40a0-4d47-8e8c-702c264388c2","commitTimeStamp":"2016-12-06T15:21:14.0655859Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2042.json","@type":"CatalogPage","commitId":"49ed7bdd-5898-4255-a43f-cf91f0087f25","commitTimeStamp":"2016-12-06T22:20:25.2882598Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2043.json","@type":"CatalogPage","commitId":"c0829d94-e4c0-4984-a704-424780cef39d","commitTimeStamp":"2016-12-07T10:11:39.3520579Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2044.json","@type":"CatalogPage","commitId":"57d883cf-ec7d-4b5b-aa08-34672629b4db","commitTimeStamp":"2016-12-07T16:25:53.5007375Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2045.json","@type":"CatalogPage","commitId":"07879985-ff39-47bf-9b53-c624edd3b1c9","commitTimeStamp":"2016-12-08T01:14:52.6198985Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2046.json","@type":"CatalogPage","commitId":"3f15e902-1559-499a-b1c7-f004a2d6d552","commitTimeStamp":"2016-12-08T12:08:35.7744758Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2047.json","@type":"CatalogPage","commitId":"97f935b2-6bf9-4666-b6ed-474012843440","commitTimeStamp":"2016-12-08T18:12:53.7456284Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2048.json","@type":"CatalogPage","commitId":"f6c2b8d7-d74e-424d-b79c-dde54a404d79","commitTimeStamp":"2016-12-09T03:04:01.7276674Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2049.json","@type":"CatalogPage","commitId":"734cb76d-687b-40cd-bf54-7e3733f22c08","commitTimeStamp":"2016-12-09T09:54:41.1721434Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2050.json","@type":"CatalogPage","commitId":"a2cf40d4-75b9-48d7-82fe-7abd7b203b04","commitTimeStamp":"2016-12-09T18:30:52.3783627Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2051.json","@type":"CatalogPage","commitId":"906db260-0d3d-4bef-8768-8cf65f1004c0","commitTimeStamp":"2016-12-10T12:51:57.5656591Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2052.json","@type":"CatalogPage","commitId":"c015e2b4-5d13-460c-b603-a4e0650d3a55","commitTimeStamp":"2016-12-11T15:08:46.6715477Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2053.json","@type":"CatalogPage","commitId":"188073d5-5c7e-4a23-af53-36350f7aa65c","commitTimeStamp":"2016-12-12T09:04:35.9759946Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2054.json","@type":"CatalogPage","commitId":"e37ef699-ef0c-4646-b6cc-1e5cd0dcadf9","commitTimeStamp":"2016-12-12T17:24:22.0372864Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2055.json","@type":"CatalogPage","commitId":"c1305344-e3d3-47b7-8de5-88f9b42c5137","commitTimeStamp":"2016-12-13T00:30:17.6143552Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2056.json","@type":"CatalogPage","commitId":"a4cdfaf9-acf4-417f-adf9-79e64fcb0107","commitTimeStamp":"2016-12-13T11:11:21.7126577Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2057.json","@type":"CatalogPage","commitId":"c90dca89-8d00-45c8-ae0f-4e118f6efd9c","commitTimeStamp":"2016-12-13T18:58:25.1488288Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2058.json","@type":"CatalogPage","commitId":"2731a137-4a25-477e-9d0e-eee63feff28e","commitTimeStamp":"2016-12-13T22:12:08.8185793Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2059.json","@type":"CatalogPage","commitId":"c185f313-1f54-4743-9d81-b68fca3b59ca","commitTimeStamp":"2016-12-13T22:57:59.9338414Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2060.json","@type":"CatalogPage","commitId":"1ff83a30-2e17-44c3-a63a-00f6de247ca0","commitTimeStamp":"2016-12-14T02:14:36.6016561Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2061.json","@type":"CatalogPage","commitId":"9e38a93f-d76a-47aa-a829-27aaed5f54ac","commitTimeStamp":"2016-12-14T04:21:11.0184698Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2062.json","@type":"CatalogPage","commitId":"968c6077-ea7c-4157-a1ea-c70e966599be","commitTimeStamp":"2016-12-14T06:27:46.5680539Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2063.json","@type":"CatalogPage","commitId":"11dbad48-458f-4ec6-9500-2622a0655cbd","commitTimeStamp":"2016-12-14T09:00:00.3332266Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2064.json","@type":"CatalogPage","commitId":"878ef43e-3821-4ff6-81f1-05fe237543ee","commitTimeStamp":"2016-12-14T17:01:59.2027153Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2065.json","@type":"CatalogPage","commitId":"d8fb6c85-d4af-41a0-9be7-1f5e201f25b1","commitTimeStamp":"2016-12-15T05:51:58.7964795Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2066.json","@type":"CatalogPage","commitId":"ebf4cbf7-92c4-492b-8a31-d034e87f62b2","commitTimeStamp":"2016-12-15T13:00:38.3609178Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2067.json","@type":"CatalogPage","commitId":"fc1fab96-d723-4053-96b5-88074a84f8e6","commitTimeStamp":"2016-12-15T18:42:10.8160975Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2068.json","@type":"CatalogPage","commitId":"e40eb11e-494b-49bf-9fc3-ca27625018b0","commitTimeStamp":"2016-12-16T06:31:51.0922651Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2069.json","@type":"CatalogPage","commitId":"9df73a7c-2f0c-4c42-a388-f62871aaf400","commitTimeStamp":"2016-12-16T15:22:09.7477871Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2070.json","@type":"CatalogPage","commitId":"5baa0676-f312-451a-a390-2756698567ee","commitTimeStamp":"2016-12-17T05:18:17.8432788Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2071.json","@type":"CatalogPage","commitId":"5323baee-17fb-46c3-b80a-ee4a318290ce","commitTimeStamp":"2016-12-18T03:42:49.6147618Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2072.json","@type":"CatalogPage","commitId":"8031186a-f0a7-47c6-b31c-8c062bc506c8","commitTimeStamp":"2016-12-19T02:06:10.5165166Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2073.json","@type":"CatalogPage","commitId":"02fb597d-e714-4c46-a71b-03138690eaa3","commitTimeStamp":"2016-12-19T14:24:14.815214Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2074.json","@type":"CatalogPage","commitId":"c775bed0-ebb4-49bd-a335-a06c5aedffff","commitTimeStamp":"2016-12-19T23:31:03.7769718Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2075.json","@type":"CatalogPage","commitId":"24785cc0-fcd5-4515-9bb2-5fab3421c909","commitTimeStamp":"2016-12-20T09:33:57.0982441Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2076.json","@type":"CatalogPage","commitId":"a842d451-a2ff-494b-a602-71b102e5538e","commitTimeStamp":"2016-12-20T17:21:57.9256922Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2077.json","@type":"CatalogPage","commitId":"c44061a0-33fd-47f8-98c8-1cbc332a4d5d","commitTimeStamp":"2016-12-21T07:39:56.8941033Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2078.json","@type":"CatalogPage","commitId":"cd64254e-bf8e-48a5-9b9a-c864e2b2b68b","commitTimeStamp":"2016-12-21T15:08:13.9903404Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2079.json","@type":"CatalogPage","commitId":"60444739-9c8d-4d5d-9581-f0623fb7d0e7","commitTimeStamp":"2016-12-21T22:50:12.2917588Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2080.json","@type":"CatalogPage","commitId":"643a3bf0-e4c2-4b89-84f0-908d5e807337","commitTimeStamp":"2016-12-22T08:50:44.4535827Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2081.json","@type":"CatalogPage","commitId":"faa337c2-5725-4c59-bca0-23870470abcf","commitTimeStamp":"2016-12-22T17:18:28.9057358Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2082.json","@type":"CatalogPage","commitId":"5b98dffb-6fd9-419d-9c73-d0fdd28697cb","commitTimeStamp":"2016-12-23T07:36:22.685103Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2083.json","@type":"CatalogPage","commitId":"de811c1b-f507-4bb9-b967-e92e50abf92f","commitTimeStamp":"2016-12-23T16:50:46.337037Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2084.json","@type":"CatalogPage","commitId":"b91998e8-145a-43cd-a313-be0e13aef8e4","commitTimeStamp":"2016-12-24T09:10:28.2223026Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2085.json","@type":"CatalogPage","commitId":"b0228a53-a63a-422e-aeed-b7397ec9e7b9","commitTimeStamp":"2016-12-25T15:54:13.2902695Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2086.json","@type":"CatalogPage","commitId":"abfe77f6-51a1-4699-a4e5-3942290c7e16","commitTimeStamp":"2016-12-26T17:57:49.4234659Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2087.json","@type":"CatalogPage","commitId":"1ee17716-c3ab-4ce8-8a3a-b1ded1a5bf81","commitTimeStamp":"2016-12-27T02:55:33.2628686Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2088.json","@type":"CatalogPage","commitId":"bd52c803-94af-41a9-b527-49f81404c331","commitTimeStamp":"2016-12-27T19:02:14.7414467Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2089.json","@type":"CatalogPage","commitId":"f1b0488e-e936-4b3b-8a34-3d59fd29d2f9","commitTimeStamp":"2016-12-28T10:41:11.0441123Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2090.json","@type":"CatalogPage","commitId":"95c0d2ea-45d9-48ae-b3ce-3d90a2c05925","commitTimeStamp":"2016-12-29T00:57:14.9739546Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2091.json","@type":"CatalogPage","commitId":"87e42ea3-1e04-4a47-bc18-7628bd89670d","commitTimeStamp":"2016-12-29T14:01:02.7959761Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2092.json","@type":"CatalogPage","commitId":"548ea363-8f86-4441-ad94-d5f0ee9a08a4","commitTimeStamp":"2016-12-30T04:28:09.7688529Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2093.json","@type":"CatalogPage","commitId":"e01cb0f3-91eb-461d-8bfe-5324a8453381","commitTimeStamp":"2016-12-30T17:16:29.2002956Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2094.json","@type":"CatalogPage","commitId":"2c549ec7-e750-400f-8d05-2ca73c70326b","commitTimeStamp":"2016-12-31T15:14:43.9491923Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2095.json","@type":"CatalogPage","commitId":"108a4e93-6bad-4c22-8849-62cd5826df8f","commitTimeStamp":"2017-01-02T08:14:25.2211131Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2096.json","@type":"CatalogPage","commitId":"156e4734-ca6f-42f7-bc5f-974e759f4242","commitTimeStamp":"2017-01-02T21:09:42.4368899Z","count":541},{"@id":"https://api.nuget.org/v3/catalog0/page2097.json","@type":"CatalogPage","commitId":"7369947d-dade-491e-8ea1-7f03e3b6f9c1","commitTimeStamp":"2017-01-03T10:40:47.6947081Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2098.json","@type":"CatalogPage","commitId":"390854d7-adad-40ba-b93e-5a6d2a29d83e","commitTimeStamp":"2017-01-03T19:41:00.0698634Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2099.json","@type":"CatalogPage","commitId":"aa35bc22-008e-41a3-b3e6-48ec30753cf8","commitTimeStamp":"2017-01-04T08:22:13.2520243Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2100.json","@type":"CatalogPage","commitId":"7b27c86f-0852-4d90-a25c-5360bd048777","commitTimeStamp":"2017-01-04T16:03:45.2300649Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2101.json","@type":"CatalogPage","commitId":"d43d0302-7436-4c58-8ab8-ba0f6cbec214","commitTimeStamp":"2017-01-05T01:55:49.194109Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2102.json","@type":"CatalogPage","commitId":"85a25311-830c-4ceb-8671-ca5bc31bfc2e","commitTimeStamp":"2017-01-05T13:46:18.2123548Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2103.json","@type":"CatalogPage","commitId":"bf958940-04c3-4f50-8971-288ad873c281","commitTimeStamp":"2017-01-05T20:52:19.4650453Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2104.json","@type":"CatalogPage","commitId":"2ef48518-1ade-4d81-a8af-e07609eff48f","commitTimeStamp":"2017-01-06T00:28:46.4294642Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2105.json","@type":"CatalogPage","commitId":"3042d206-c091-4bff-8088-33184f5f4eac","commitTimeStamp":"2017-01-06T08:48:21.3539932Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2106.json","@type":"CatalogPage","commitId":"9fcd35b3-7f3d-4706-af4e-3e952eb13bfe","commitTimeStamp":"2017-01-06T20:12:08.2940435Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2107.json","@type":"CatalogPage","commitId":"ab955f79-f29c-4e5d-96f5-2bc18d960ea7","commitTimeStamp":"2017-01-07T14:41:06.2829481Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2108.json","@type":"CatalogPage","commitId":"39e79b2f-ea65-441d-9954-5599b7cad23b","commitTimeStamp":"2017-01-08T12:34:04.2593058Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2109.json","@type":"CatalogPage","commitId":"96b4bca7-bdda-4a23-84ef-f178ffaca6b0","commitTimeStamp":"2017-01-08T23:34:27.7402575Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2110.json","@type":"CatalogPage","commitId":"1ef4cda2-ca63-4ee2-be10-70b2667f3750","commitTimeStamp":"2017-01-09T12:31:53.3300243Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2111.json","@type":"CatalogPage","commitId":"9e9da444-5667-46ef-b09f-2924e8b41e19","commitTimeStamp":"2017-01-09T21:00:12.290369Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2112.json","@type":"CatalogPage","commitId":"937136b1-ff9c-4cde-a439-58c0f7d5708f","commitTimeStamp":"2017-01-10T08:55:33.3218951Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2113.json","@type":"CatalogPage","commitId":"bbc88259-d6ae-496a-a022-e58eff29e631","commitTimeStamp":"2017-01-10T13:23:32.8904557Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2114.json","@type":"CatalogPage","commitId":"0d85cc06-a2a1-4fcb-9503-2b8eda3acc49","commitTimeStamp":"2017-01-10T22:30:10.0164216Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2115.json","@type":"CatalogPage","commitId":"374a4b80-7eca-4d58-961b-be76af7ff337","commitTimeStamp":"2017-01-11T09:59:12.5335039Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2116.json","@type":"CatalogPage","commitId":"1eaad6f0-e07b-41f8-9e75-801be151cdb0","commitTimeStamp":"2017-01-11T20:00:36.7028043Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2117.json","@type":"CatalogPage","commitId":"9f2ea6f6-5ca3-48c5-bf0e-68e461a7b8a7","commitTimeStamp":"2017-01-12T09:07:13.8096653Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2118.json","@type":"CatalogPage","commitId":"2372ae3f-bcf6-415f-a045-5a99a8ffe78f","commitTimeStamp":"2017-01-12T18:16:18.3284303Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2119.json","@type":"CatalogPage","commitId":"967ed0df-6450-4e48-a80a-e43f9f469b59","commitTimeStamp":"2017-01-13T10:00:18.0430217Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2120.json","@type":"CatalogPage","commitId":"9d49339f-c64c-40f7-aa10-fc37ec1080ce","commitTimeStamp":"2017-01-13T18:21:39.0755015Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2121.json","@type":"CatalogPage","commitId":"7373ff70-520d-452f-8785-ae630bbbe4b0","commitTimeStamp":"2017-01-14T12:51:49.7867115Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2122.json","@type":"CatalogPage","commitId":"eafcf474-b384-4917-8607-4640fb348ffd","commitTimeStamp":"2017-01-15T03:45:34.468375Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2123.json","@type":"CatalogPage","commitId":"d2e88d4f-1bf6-4f81-b1fb-be434bbb2290","commitTimeStamp":"2017-01-16T04:09:50.1034837Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2124.json","@type":"CatalogPage","commitId":"f701e626-5092-4688-a0e6-b4fafa64f358","commitTimeStamp":"2017-01-16T12:53:54.5867326Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2125.json","@type":"CatalogPage","commitId":"04ff96f6-97f1-4f8e-9b1e-78523259ee2a","commitTimeStamp":"2017-01-16T23:49:13.4098141Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2126.json","@type":"CatalogPage","commitId":"c23c7574-ec95-4f76-ab5b-c2e508995f03","commitTimeStamp":"2017-01-17T09:39:36.3151785Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2127.json","@type":"CatalogPage","commitId":"3bcbec17-cf0a-465b-806e-83be1510b6d1","commitTimeStamp":"2017-01-17T17:05:49.546719Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2128.json","@type":"CatalogPage","commitId":"a9001972-648b-4da0-8af0-a21a2ff14ed7","commitTimeStamp":"2017-01-18T07:06:57.2353509Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2129.json","@type":"CatalogPage","commitId":"5fdacd59-c7db-49d8-b004-3143f87b22e0","commitTimeStamp":"2017-01-18T14:26:07.6555894Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2130.json","@type":"CatalogPage","commitId":"d07fcd69-fd39-40c2-b21c-98c855922e7f","commitTimeStamp":"2017-01-19T00:20:09.4548049Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2131.json","@type":"CatalogPage","commitId":"f48cc6cf-c7f4-41da-96d0-13be692577cb","commitTimeStamp":"2017-01-19T09:53:30.7151052Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2132.json","@type":"CatalogPage","commitId":"cfdfabd5-58a5-447b-86ba-c875ed514f5b","commitTimeStamp":"2017-01-19T15:57:25.6651391Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2133.json","@type":"CatalogPage","commitId":"7e197386-6415-44ed-880e-dcb6ff7e2376","commitTimeStamp":"2017-01-20T06:24:21.9428786Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2134.json","@type":"CatalogPage","commitId":"f521ad1d-80f0-4141-a1b0-dd69cc0d4367","commitTimeStamp":"2017-01-20T13:08:37.9887427Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2135.json","@type":"CatalogPage","commitId":"9425ebb7-9314-48a2-989f-923f42384e30","commitTimeStamp":"2017-01-20T21:12:58.8727763Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2136.json","@type":"CatalogPage","commitId":"518eeca6-2a57-4d50-ae22-c52ec7189072","commitTimeStamp":"2017-01-21T06:45:02.6981408Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2137.json","@type":"CatalogPage","commitId":"eeb200b9-8554-4cdb-930b-2a7a5988b2b1","commitTimeStamp":"2017-01-22T01:17:34.6288037Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2138.json","@type":"CatalogPage","commitId":"a9f369d9-1e3e-4db0-815c-d12c308d9863","commitTimeStamp":"2017-01-22T21:17:59.2924504Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2139.json","@type":"CatalogPage","commitId":"fd070719-1702-4127-9392-bf0689bc8523","commitTimeStamp":"2017-01-23T11:37:53.3148278Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2140.json","@type":"CatalogPage","commitId":"b9ee6c57-781e-4763-9b7e-71a0d62612cd","commitTimeStamp":"2017-01-23T20:17:00.4977116Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2141.json","@type":"CatalogPage","commitId":"d61a66d6-5cd3-453d-bb81-11ea17d0cc2e","commitTimeStamp":"2017-01-24T09:10:36.4785324Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2142.json","@type":"CatalogPage","commitId":"73dd6af5-8d1e-480a-b117-a96db354fb70","commitTimeStamp":"2017-01-24T16:11:27.972989Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2143.json","@type":"CatalogPage","commitId":"fdd371ee-fcc4-431b-9219-cc69a1122077","commitTimeStamp":"2017-01-24T21:57:47.3399333Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2144.json","@type":"CatalogPage","commitId":"36c68f66-19a6-41c1-92ee-e644ae51e935","commitTimeStamp":"2017-01-25T09:11:26.3081412Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2145.json","@type":"CatalogPage","commitId":"e1ed4681-a63c-45d5-bf61-90f13905fa8d","commitTimeStamp":"2017-01-25T15:52:18.5377342Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2146.json","@type":"CatalogPage","commitId":"3c577e5c-a2ad-4e50-89ae-d280d5e0ef8c","commitTimeStamp":"2017-01-25T22:39:05.3179482Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2147.json","@type":"CatalogPage","commitId":"4503e6c2-f4e6-40dc-8261-bb811471ef88","commitTimeStamp":"2017-01-26T08:13:44.5412768Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2148.json","@type":"CatalogPage","commitId":"6e0b3f94-e9d3-4303-9eae-d08730d9e4f7","commitTimeStamp":"2017-01-26T14:35:18.2211779Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2149.json","@type":"CatalogPage","commitId":"d74f229d-ec52-4416-9bc9-38ef1f8d1536","commitTimeStamp":"2017-01-26T21:12:17.0487805Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2150.json","@type":"CatalogPage","commitId":"39336b15-d4ef-4b43-9023-d84eb718a821","commitTimeStamp":"2017-01-27T06:59:25.8282718Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2151.json","@type":"CatalogPage","commitId":"1974a85d-d675-4441-baff-d69400242605","commitTimeStamp":"2017-01-27T16:15:37.5089256Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2152.json","@type":"CatalogPage","commitId":"cfdcff60-3b5a-474c-a66e-628ffc9fa285","commitTimeStamp":"2017-01-28T14:21:53.9582671Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2153.json","@type":"CatalogPage","commitId":"65465b55-ba15-4728-bb35-fc333250f944","commitTimeStamp":"2017-01-29T12:28:48.7000237Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2154.json","@type":"CatalogPage","commitId":"98d00b68-b20e-499c-a3af-742f8b1ebb59","commitTimeStamp":"2017-01-30T00:15:31.6087175Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2155.json","@type":"CatalogPage","commitId":"b9ad7f5c-6805-4cbc-9ef3-b9092bfed9b9","commitTimeStamp":"2017-01-30T11:22:24.6378082Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2156.json","@type":"CatalogPage","commitId":"63660eba-8a6d-4ffe-8eab-9d6cf4c7d3b2","commitTimeStamp":"2017-01-30T19:50:51.7459034Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2157.json","@type":"CatalogPage","commitId":"7798b466-24df-4461-927a-a75fb14547dc","commitTimeStamp":"2017-01-31T07:54:08.0252512Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2158.json","@type":"CatalogPage","commitId":"1f889991-1b10-4ce7-8a8e-53fbd24c52fb","commitTimeStamp":"2017-01-31T13:18:09.8219547Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2159.json","@type":"CatalogPage","commitId":"1871f092-8fe8-42b2-b5d0-d5ee49cc1aaf","commitTimeStamp":"2017-01-31T19:51:18.4451189Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2160.json","@type":"CatalogPage","commitId":"7d496c5b-66c7-4446-beab-a98f1856dec4","commitTimeStamp":"2017-02-01T05:04:32.7700789Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2161.json","@type":"CatalogPage","commitId":"c31ddcf2-fe19-4aa1-9d47-e8f33753f6f7","commitTimeStamp":"2017-02-01T12:24:14.2042622Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2162.json","@type":"CatalogPage","commitId":"a1909bf0-9e31-4a40-aa35-6afc2305620e","commitTimeStamp":"2017-02-01T19:32:30.3427705Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2163.json","@type":"CatalogPage","commitId":"d024543d-59a6-4802-ab1d-8f9a14be0a3d","commitTimeStamp":"2017-02-02T09:44:33.2617008Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2164.json","@type":"CatalogPage","commitId":"f2186232-1841-49dd-ad5b-f054ddfccdcc","commitTimeStamp":"2017-02-02T17:36:56.0553462Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2165.json","@type":"CatalogPage","commitId":"60ac4bd7-6bf0-452b-815a-5aa69161db1b","commitTimeStamp":"2017-02-03T03:51:13.3063732Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2166.json","@type":"CatalogPage","commitId":"1e1ca4fd-98be-4621-a6d2-0146f9ad7a03","commitTimeStamp":"2017-02-03T09:49:02.4795653Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2167.json","@type":"CatalogPage","commitId":"94f37e40-1c4a-4ca5-89d2-0f0e51931211","commitTimeStamp":"2017-02-03T16:14:56.5412853Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2168.json","@type":"CatalogPage","commitId":"f69693b9-8a26-49c1-9d8b-98e13088cfcd","commitTimeStamp":"2017-02-04T02:36:16.8850831Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2169.json","@type":"CatalogPage","commitId":"5e1e1518-3822-4084-b4aa-ff83c296fbce","commitTimeStamp":"2017-02-04T18:28:05.44389Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2170.json","@type":"CatalogPage","commitId":"16e29b89-f774-4c1f-9c81-45574af1ce69","commitTimeStamp":"2017-02-05T01:03:39.1510602Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2171.json","@type":"CatalogPage","commitId":"d3639b47-160f-40ca-8c3b-9f9e80d4cbb4","commitTimeStamp":"2017-02-05T17:10:35.8880145Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2172.json","@type":"CatalogPage","commitId":"ab642d41-4073-4047-911e-22b063141fe0","commitTimeStamp":"2017-02-06T06:49:32.3565206Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2173.json","@type":"CatalogPage","commitId":"bb895116-3d74-43e7-b594-b90e517cedc1","commitTimeStamp":"2017-02-06T15:41:10.2661332Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2174.json","@type":"CatalogPage","commitId":"098e1815-6817-4ae0-abee-0db38228fe69","commitTimeStamp":"2017-02-07T02:45:18.8002824Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2175.json","@type":"CatalogPage","commitId":"f0e47958-b235-4abc-b198-2fc78e58418d","commitTimeStamp":"2017-02-07T15:07:58.7638851Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2176.json","@type":"CatalogPage","commitId":"e9726976-fb3f-4547-9191-68ff136a4ce0","commitTimeStamp":"2017-02-07T22:09:56.0534575Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2177.json","@type":"CatalogPage","commitId":"ee8e58dc-89f2-4404-8300-21c1435b32a4","commitTimeStamp":"2017-02-08T08:52:43.2858955Z","count":534},{"@id":"https://api.nuget.org/v3/catalog0/page2178.json","@type":"CatalogPage","commitId":"ce23d723-0b5b-4067-a1c0-ff6c4481f87d","commitTimeStamp":"2017-02-08T09:01:08.2471824Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2179.json","@type":"CatalogPage","commitId":"8db31325-1a9a-4475-aebd-e9db9d941297","commitTimeStamp":"2017-02-08T09:07:42.744482Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2180.json","@type":"CatalogPage","commitId":"a86b733a-49ac-49a2-9dea-89bb69d1bf05","commitTimeStamp":"2017-02-08T09:13:35.0788009Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2181.json","@type":"CatalogPage","commitId":"e3793bde-5d50-4d1e-b2d9-a8419877f0a1","commitTimeStamp":"2017-02-08T09:19:36.5845692Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2182.json","@type":"CatalogPage","commitId":"2639445a-bb3b-411c-806e-65bf224c22d1","commitTimeStamp":"2017-02-08T09:25:30.2445737Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2183.json","@type":"CatalogPage","commitId":"0fbaf0c9-876f-4712-a788-2e1356f39836","commitTimeStamp":"2017-02-08T09:31:19.5124119Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2184.json","@type":"CatalogPage","commitId":"3ac11a6f-6b60-4ff6-89e0-1412114032a6","commitTimeStamp":"2017-02-08T09:37:03.450381Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2185.json","@type":"CatalogPage","commitId":"b85a1344-7585-4018-a228-4e45c0b64470","commitTimeStamp":"2017-02-08T09:42:43.4932559Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2186.json","@type":"CatalogPage","commitId":"ef03bedc-29da-4487-a8d2-68c7203ee136","commitTimeStamp":"2017-02-08T09:48:26.5645034Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2187.json","@type":"CatalogPage","commitId":"fa38143d-5708-4a52-ae0f-b0e256d47c7c","commitTimeStamp":"2017-02-08T09:54:06.906098Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2188.json","@type":"CatalogPage","commitId":"440a630a-a2ce-467b-b16f-26fb6bf3f3cb","commitTimeStamp":"2017-02-08T09:59:58.7769026Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2189.json","@type":"CatalogPage","commitId":"11c5677d-372b-458f-bc10-1b618b4e06e9","commitTimeStamp":"2017-02-08T10:05:52.7582151Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2190.json","@type":"CatalogPage","commitId":"f7980e50-71b3-4416-9642-cfd4e180fde7","commitTimeStamp":"2017-02-08T10:11:32.7851576Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2191.json","@type":"CatalogPage","commitId":"280de7a0-ac1f-4cad-b968-1ae0bcf3be03","commitTimeStamp":"2017-02-08T10:17:13.3461954Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2192.json","@type":"CatalogPage","commitId":"026433d7-ad54-47b8-a156-9647e9c8eda9","commitTimeStamp":"2017-02-08T10:23:00.7709279Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2193.json","@type":"CatalogPage","commitId":"90514bb8-0441-437e-a431-d91938d44417","commitTimeStamp":"2017-02-08T10:28:53.4425651Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2194.json","@type":"CatalogPage","commitId":"9ecf4934-3403-4f67-8bf1-ac0caff8d3a3","commitTimeStamp":"2017-02-08T10:34:51.959105Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2195.json","@type":"CatalogPage","commitId":"9cfd88e6-0312-48e1-a790-1b348c98421d","commitTimeStamp":"2017-02-08T10:40:43.1057197Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2196.json","@type":"CatalogPage","commitId":"a6219783-58a0-44f7-85de-6c8e6a657319","commitTimeStamp":"2017-02-08T13:26:26.2121658Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2197.json","@type":"CatalogPage","commitId":"708f2aa5-052c-4144-9a38-11a821c4f02a","commitTimeStamp":"2017-02-08T20:28:20.4101601Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2198.json","@type":"CatalogPage","commitId":"22b9bf1a-9361-4c6a-bbac-9c80ccb944f9","commitTimeStamp":"2017-02-09T01:19:17.1167717Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2199.json","@type":"CatalogPage","commitId":"b2a528e7-f88a-42e4-8246-ef8308e99c51","commitTimeStamp":"2017-02-09T06:46:31.5430647Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2200.json","@type":"CatalogPage","commitId":"4e1e6849-35db-4a30-931f-5fe3413845f0","commitTimeStamp":"2017-02-09T10:32:26.0889433Z","count":533},{"@id":"https://api.nuget.org/v3/catalog0/page2201.json","@type":"CatalogPage","commitId":"63fa033c-4444-4018-a020-a7bf020fbf4f","commitTimeStamp":"2017-02-09T15:28:36.3728981Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2202.json","@type":"CatalogPage","commitId":"add1faae-b3b1-4db6-a6eb-c049cd70d707","commitTimeStamp":"2017-02-09T21:36:43.8491203Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page2203.json","@type":"CatalogPage","commitId":"4fa020ad-552a-4eb5-adaa-7f7c4899b041","commitTimeStamp":"2017-02-10T09:06:47.7738365Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2204.json","@type":"CatalogPage","commitId":"42a90e5e-5f0e-4e1b-a5ca-c5c1a70fa3fc","commitTimeStamp":"2017-02-10T14:46:25.3077385Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2205.json","@type":"CatalogPage","commitId":"c48b2de8-7865-4cb0-8f8d-bcedb9f342ba","commitTimeStamp":"2017-02-10T21:03:29.0287026Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2206.json","@type":"CatalogPage","commitId":"64dd9e71-a70e-443d-9f1b-f5ea9b992e80","commitTimeStamp":"2017-02-11T15:04:50.5113045Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2207.json","@type":"CatalogPage","commitId":"fe6d8ab6-e0dc-44bc-907b-e22d48de1ed0","commitTimeStamp":"2017-02-12T08:19:01.4949908Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2208.json","@type":"CatalogPage","commitId":"c648ac52-ed67-4ace-97a3-02c27ccf1111","commitTimeStamp":"2017-02-13T02:52:52.4686936Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2209.json","@type":"CatalogPage","commitId":"5e5b3841-f2da-493e-af65-6ae7e72ba969","commitTimeStamp":"2017-02-13T11:24:38.157507Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2210.json","@type":"CatalogPage","commitId":"68caa5d4-ca50-49d4-88b1-ef7d14b6c2f6","commitTimeStamp":"2017-02-13T17:16:07.6933782Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2211.json","@type":"CatalogPage","commitId":"02d3f794-c3f1-4eab-81a0-6017cdfa4974","commitTimeStamp":"2017-02-14T05:06:46.7738846Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2212.json","@type":"CatalogPage","commitId":"32df8323-c572-4cdb-9ebf-7729083918f3","commitTimeStamp":"2017-02-14T14:04:17.5613997Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2213.json","@type":"CatalogPage","commitId":"1c148753-f906-430d-ba18-f85d3303f15d","commitTimeStamp":"2017-02-14T21:21:07.0505289Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2214.json","@type":"CatalogPage","commitId":"5f41e04d-8f5d-430e-962e-27e13df1e3f4","commitTimeStamp":"2017-02-15T05:08:33.4944037Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2215.json","@type":"CatalogPage","commitId":"e5d713a7-c9c3-4245-b754-7f6cf64f5abe","commitTimeStamp":"2017-02-15T14:47:09.8344026Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2216.json","@type":"CatalogPage","commitId":"d3443829-5abb-4b9d-bb70-f4dc1bbcfd77","commitTimeStamp":"2017-02-15T21:36:15.9015589Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2217.json","@type":"CatalogPage","commitId":"2c4f38a1-4d5b-4e16-a7d8-44fa16c939df","commitTimeStamp":"2017-02-16T12:20:43.8877991Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2218.json","@type":"CatalogPage","commitId":"ccf4bbb9-9c8c-4cd7-8557-46806204b21a","commitTimeStamp":"2017-02-16T20:19:19.8834276Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2219.json","@type":"CatalogPage","commitId":"b97b1c4c-f3a0-4ec8-b887-750ed8eaf1f7","commitTimeStamp":"2017-02-17T07:08:31.1704954Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2220.json","@type":"CatalogPage","commitId":"d53a0062-0cac-4744-bc16-ba45b264c94e","commitTimeStamp":"2017-02-17T12:56:44.454978Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2221.json","@type":"CatalogPage","commitId":"3580cecb-f27b-4abd-914a-d61c43e305e0","commitTimeStamp":"2017-02-17T21:02:31.2908854Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2222.json","@type":"CatalogPage","commitId":"31d265b1-5a3e-4001-9542-48e8a1359215","commitTimeStamp":"2017-02-18T09:23:04.8603042Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2223.json","@type":"CatalogPage","commitId":"f5bf942e-39c3-4701-9868-837f1ae891b1","commitTimeStamp":"2017-02-19T00:39:59.5381207Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2224.json","@type":"CatalogPage","commitId":"bbfd8c80-107e-42e8-9345-bf6bf5776496","commitTimeStamp":"2017-02-19T16:38:11.3427935Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2225.json","@type":"CatalogPage","commitId":"3e1001b6-fd80-4e4e-8001-8fa32c741812","commitTimeStamp":"2017-02-20T09:01:39.2058144Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2226.json","@type":"CatalogPage","commitId":"17065895-7a2c-4638-8417-47aa760e28b7","commitTimeStamp":"2017-02-20T14:47:55.0034111Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2227.json","@type":"CatalogPage","commitId":"f399cf08-f47e-4498-bc44-81a7a33c7a74","commitTimeStamp":"2017-02-20T20:56:59.5759906Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2228.json","@type":"CatalogPage","commitId":"3c894ca0-ac03-4592-b224-3e47af525291","commitTimeStamp":"2017-02-21T08:45:43.4614961Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2229.json","@type":"CatalogPage","commitId":"0800e5d1-22fc-4511-a0b4-7503d6e0d234","commitTimeStamp":"2017-02-21T18:23:05.2108537Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2230.json","@type":"CatalogPage","commitId":"203e9edb-eaad-4015-af3c-7f36918a62ae","commitTimeStamp":"2017-02-22T03:11:07.2832144Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2231.json","@type":"CatalogPage","commitId":"162cc900-7b4e-4dc4-85b6-f4bcbe1109a0","commitTimeStamp":"2017-02-22T10:58:30.0774917Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2232.json","@type":"CatalogPage","commitId":"ce7c7391-befd-4190-95fd-75a43a11afa9","commitTimeStamp":"2017-02-22T17:58:03.5608873Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2233.json","@type":"CatalogPage","commitId":"66792876-7508-4023-9418-a0705614524b","commitTimeStamp":"2017-02-23T10:29:47.2755766Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2234.json","@type":"CatalogPage","commitId":"a9951299-b4b1-441b-be88-011fd8f645e9","commitTimeStamp":"2017-02-23T18:45:22.0401615Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2235.json","@type":"CatalogPage","commitId":"a8ed0874-564e-4632-ba5d-ee5ce200ebc9","commitTimeStamp":"2017-02-24T05:17:43.9429528Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2236.json","@type":"CatalogPage","commitId":"43a6c9bc-7668-4a15-b291-8aaf2b0d4708","commitTimeStamp":"2017-02-24T14:24:20.9848315Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2237.json","@type":"CatalogPage","commitId":"7daed8de-c2b8-488b-ae49-ae6b947bacee","commitTimeStamp":"2017-02-24T23:16:36.8736541Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2238.json","@type":"CatalogPage","commitId":"ca6fe2d4-006e-48a3-a4a8-d26d384fbfb8","commitTimeStamp":"2017-02-25T11:34:09.6330278Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2239.json","@type":"CatalogPage","commitId":"50487e0b-7204-493e-b39e-8b28846e8689","commitTimeStamp":"2017-02-25T17:53:40.248531Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2240.json","@type":"CatalogPage","commitId":"82a9c581-8f9c-408d-a578-25b41f03c4e5","commitTimeStamp":"2017-02-26T08:00:24.3208611Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2241.json","@type":"CatalogPage","commitId":"638b5480-90b8-4a78-8093-224104e796ae","commitTimeStamp":"2017-02-26T21:28:18.4391328Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2242.json","@type":"CatalogPage","commitId":"38ba8177-b739-4c0b-81f0-e6efd9ba8adb","commitTimeStamp":"2017-02-27T09:24:03.3705648Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2243.json","@type":"CatalogPage","commitId":"a2546e02-758a-4fd3-b94d-e40f3cadddb3","commitTimeStamp":"2017-02-27T16:04:35.091998Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2244.json","@type":"CatalogPage","commitId":"2c71ed51-b7a8-40c0-a67e-51faebf194b3","commitTimeStamp":"2017-02-27T21:51:33.5626434Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2245.json","@type":"CatalogPage","commitId":"ad87e854-a3c7-496a-97a6-30e4a675c538","commitTimeStamp":"2017-02-28T05:56:27.2906859Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2246.json","@type":"CatalogPage","commitId":"29850d3d-76b7-4736-8807-23e9536dfe91","commitTimeStamp":"2017-02-28T11:20:43.4555509Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2247.json","@type":"CatalogPage","commitId":"4222a48f-e24d-487e-81bc-e9cd5f4e3d97","commitTimeStamp":"2017-02-28T16:10:32.9744894Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2248.json","@type":"CatalogPage","commitId":"36a22550-4e06-4ad1-bd3a-13aefe7b30a2","commitTimeStamp":"2017-03-01T03:46:23.3640741Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2249.json","@type":"CatalogPage","commitId":"6dc42c55-21b9-4a1d-88c3-aebacb1ea01b","commitTimeStamp":"2017-03-01T13:38:17.2130488Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2250.json","@type":"CatalogPage","commitId":"27b3e604-05c5-4468-9319-782777504176","commitTimeStamp":"2017-03-01T22:47:47.4618639Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2251.json","@type":"CatalogPage","commitId":"0f5740d6-22df-4185-b096-0e22f391236b","commitTimeStamp":"2017-03-02T10:05:46.4219645Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2252.json","@type":"CatalogPage","commitId":"0a5d2aca-ce6d-442c-99b7-bc122d2e509c","commitTimeStamp":"2017-03-02T17:57:02.589806Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2253.json","@type":"CatalogPage","commitId":"def87b27-1f47-4ed1-b218-44667f3ce20a","commitTimeStamp":"2017-03-03T06:40:48.0039801Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2254.json","@type":"CatalogPage","commitId":"6cc40ae3-c96c-484b-ac40-46cb6564dc30","commitTimeStamp":"2017-03-03T15:28:54.086141Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2255.json","@type":"CatalogPage","commitId":"924a6857-3832-41db-84b4-bc85b7ebb989","commitTimeStamp":"2017-03-04T03:16:32.509248Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2256.json","@type":"CatalogPage","commitId":"6e7f969c-bf5b-4586-8193-b639f15b871f","commitTimeStamp":"2017-03-05T03:42:16.0118597Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2257.json","@type":"CatalogPage","commitId":"44361eb4-87c0-428f-b161-5cc51970e9f0","commitTimeStamp":"2017-03-06T02:50:41.7886671Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2258.json","@type":"CatalogPage","commitId":"6466f414-8910-4137-9b51-225079b33a69","commitTimeStamp":"2017-03-06T13:05:25.7258754Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2259.json","@type":"CatalogPage","commitId":"6f54fd49-0e08-487d-8c61-6bfeae3a2c8c","commitTimeStamp":"2017-03-06T18:37:42.7710106Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2260.json","@type":"CatalogPage","commitId":"a7a1c3a1-13f9-4adf-ab69-c41a6be54540","commitTimeStamp":"2017-03-07T00:54:34.2982562Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2261.json","@type":"CatalogPage","commitId":"e9f1ca6d-a0cf-4c65-9c46-f3f32419726c","commitTimeStamp":"2017-03-07T10:47:51.6067887Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2262.json","@type":"CatalogPage","commitId":"72001ee0-79ef-440f-b2be-9b335225329b","commitTimeStamp":"2017-03-07T21:05:03.1049833Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2263.json","@type":"CatalogPage","commitId":"9f20bbf5-e40f-45db-b3a2-6db7a9d8e040","commitTimeStamp":"2017-03-08T05:04:58.406806Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2264.json","@type":"CatalogPage","commitId":"f22908e8-380b-4256-a93e-0c7567f3dc96","commitTimeStamp":"2017-03-08T10:58:03.7355908Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2265.json","@type":"CatalogPage","commitId":"58e80a03-567d-47f8-97f3-60ce9b333ad3","commitTimeStamp":"2017-03-08T17:05:36.9776857Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2266.json","@type":"CatalogPage","commitId":"ef9ef97b-b45e-488c-92ec-d39d6ba4bf29","commitTimeStamp":"2017-03-09T03:48:21.8775916Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2267.json","@type":"CatalogPage","commitId":"47546770-6860-496d-8cfd-49d8f6d1b27b","commitTimeStamp":"2017-03-09T14:10:41.9774925Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2268.json","@type":"CatalogPage","commitId":"09e5cdb6-577b-4bea-a543-49fc02f80f92","commitTimeStamp":"2017-03-09T20:15:11.7943165Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2269.json","@type":"CatalogPage","commitId":"90f53253-2bc1-41d7-8510-d4493c7e67b4","commitTimeStamp":"2017-03-10T06:03:02.6867625Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2270.json","@type":"CatalogPage","commitId":"ef66f787-13f7-4b81-b7fe-ed0e33c913b2","commitTimeStamp":"2017-03-10T13:17:25.6937292Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2271.json","@type":"CatalogPage","commitId":"b5bbae09-152d-49ea-b565-10ed03366501","commitTimeStamp":"2017-03-10T23:27:49.8813943Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2272.json","@type":"CatalogPage","commitId":"04421686-1e94-495c-afca-11b84e54a1e2","commitTimeStamp":"2017-03-11T09:30:08.7836353Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2273.json","@type":"CatalogPage","commitId":"0605af88-b102-46ae-b34f-5b1093786006","commitTimeStamp":"2017-03-11T19:38:37.9667447Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2274.json","@type":"CatalogPage","commitId":"e3e9d1a2-3b98-487e-9857-49087d6685df","commitTimeStamp":"2017-03-12T14:23:16.1085697Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2275.json","@type":"CatalogPage","commitId":"c962e78c-34cf-4406-a54d-aecedf24a555","commitTimeStamp":"2017-03-13T03:27:22.1290864Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2276.json","@type":"CatalogPage","commitId":"862ee81f-d294-4d60-9eb8-5f112ba2e913","commitTimeStamp":"2017-03-13T12:36:42.6952955Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2277.json","@type":"CatalogPage","commitId":"a95ab90a-7bce-477d-b290-fa2c90e2b088","commitTimeStamp":"2017-03-13T18:20:18.8472842Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2278.json","@type":"CatalogPage","commitId":"4200baea-46af-4a8e-a3d9-df20f3a5b040","commitTimeStamp":"2017-03-14T06:57:30.7874434Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2279.json","@type":"CatalogPage","commitId":"6c08535b-bbdd-4b42-bc67-455fbef3ff10","commitTimeStamp":"2017-03-14T14:00:26.4105054Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2280.json","@type":"CatalogPage","commitId":"40ef4c8e-5067-4329-ab8e-ff537479cf59","commitTimeStamp":"2017-03-14T21:35:48.4324389Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2281.json","@type":"CatalogPage","commitId":"571c8a65-4895-4c4f-9bb9-2ff2f9b1018c","commitTimeStamp":"2017-03-15T07:23:36.3532472Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2282.json","@type":"CatalogPage","commitId":"ebb5d3e0-a951-42b4-81db-0cc8a36af35f","commitTimeStamp":"2017-03-15T13:48:47.0416959Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2283.json","@type":"CatalogPage","commitId":"dd3aa402-d764-484d-9c4f-1f1e1e20e229","commitTimeStamp":"2017-03-15T21:15:33.2322876Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2284.json","@type":"CatalogPage","commitId":"7d3f0798-8768-431a-90aa-3d8ec3855174","commitTimeStamp":"2017-03-16T08:47:51.0102259Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2285.json","@type":"CatalogPage","commitId":"5a7ce234-7a62-4ec2-80d8-be75dbeafa62","commitTimeStamp":"2017-03-16T17:11:20.0557742Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2286.json","@type":"CatalogPage","commitId":"c3d1b1ea-367b-486a-869d-4836c4c9cdb6","commitTimeStamp":"2017-03-17T01:17:15.9435625Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2287.json","@type":"CatalogPage","commitId":"39e57e02-a24a-4dff-a58e-5e1dcc97bc8e","commitTimeStamp":"2017-03-17T12:29:04.5948886Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2288.json","@type":"CatalogPage","commitId":"057236ec-103b-4e85-8240-11c9d34ac671","commitTimeStamp":"2017-03-17T22:05:13.8730182Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2289.json","@type":"CatalogPage","commitId":"fe719e51-8208-4d4d-b22f-e8d249cfc3ac","commitTimeStamp":"2017-03-18T16:28:15.9115988Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2290.json","@type":"CatalogPage","commitId":"6108cfda-655c-4d30-a413-1883a0699b8d","commitTimeStamp":"2017-03-19T10:32:58.2878206Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2291.json","@type":"CatalogPage","commitId":"bba5c485-bc40-449f-8578-25b66fef9b17","commitTimeStamp":"2017-03-19T20:43:03.3122351Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2292.json","@type":"CatalogPage","commitId":"5729070c-3dc6-4f94-ab96-457a74877c65","commitTimeStamp":"2017-03-20T08:32:12.2681815Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2293.json","@type":"CatalogPage","commitId":"f5e725ec-413a-4ec6-968c-213e6a7fa45b","commitTimeStamp":"2017-03-20T17:00:07.4043773Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2294.json","@type":"CatalogPage","commitId":"2e55962f-e418-4e07-b6d7-87e04d540a7e","commitTimeStamp":"2017-03-20T22:55:37.9958072Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2295.json","@type":"CatalogPage","commitId":"a71feba0-0b8c-4780-98ef-b6e842dcd1da","commitTimeStamp":"2017-03-21T09:46:32.5693489Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2296.json","@type":"CatalogPage","commitId":"32f147c7-8950-4f98-811d-ef03d07af5fd","commitTimeStamp":"2017-03-21T17:10:07.2550987Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2297.json","@type":"CatalogPage","commitId":"4f574df4-a6c3-44d0-88a2-88544fd05c32","commitTimeStamp":"2017-03-22T00:01:30.9619056Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2298.json","@type":"CatalogPage","commitId":"222a53f8-0d9f-4722-8ee6-44cb15907e49","commitTimeStamp":"2017-03-22T11:49:24.4213264Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2299.json","@type":"CatalogPage","commitId":"eba42908-1091-472c-8bcf-a6e7be6c2a96","commitTimeStamp":"2017-03-22T18:26:33.6039909Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2300.json","@type":"CatalogPage","commitId":"48e4c915-8dd4-4a36-880a-8b9250e9500e","commitTimeStamp":"2017-03-23T06:52:26.7234859Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2301.json","@type":"CatalogPage","commitId":"c0c5f92c-6ac4-4ced-a94f-d8789ac53e82","commitTimeStamp":"2017-03-23T14:53:30.2327961Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2302.json","@type":"CatalogPage","commitId":"404bd280-afa7-4039-a49f-19d03ba4d1c4","commitTimeStamp":"2017-03-23T21:19:59.0228989Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2303.json","@type":"CatalogPage","commitId":"44de76c0-06e6-4c02-bb5d-176b7cc89df3","commitTimeStamp":"2017-03-24T08:30:21.9605827Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2304.json","@type":"CatalogPage","commitId":"8e2a2ac7-2e0c-4d4f-8f94-f169c4c587de","commitTimeStamp":"2017-03-24T17:19:16.4162346Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2305.json","@type":"CatalogPage","commitId":"f8f73fa5-288d-49a7-a35f-41dd4c601747","commitTimeStamp":"2017-03-25T04:39:00.0034767Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2306.json","@type":"CatalogPage","commitId":"df6f045f-da7e-4426-95da-9f364090a457","commitTimeStamp":"2017-03-25T19:57:38.5130359Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2307.json","@type":"CatalogPage","commitId":"ab5b1490-6997-48a6-b7d1-4f541dcd79bd","commitTimeStamp":"2017-03-26T15:23:25.1742247Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2308.json","@type":"CatalogPage","commitId":"28853cc6-92de-4ca8-8e06-231f9acafa89","commitTimeStamp":"2017-03-27T05:22:51.0722686Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page2309.json","@type":"CatalogPage","commitId":"d325bef0-bc20-421a-8e86-91e343494e95","commitTimeStamp":"2017-03-27T13:54:37.6708141Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2310.json","@type":"CatalogPage","commitId":"cd458508-095c-4dc4-84c3-b4a05b8b4661","commitTimeStamp":"2017-03-27T19:56:53.9762637Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2311.json","@type":"CatalogPage","commitId":"2b19076f-2bb3-4366-9c98-74ec0658c28e","commitTimeStamp":"2017-03-28T06:01:45.917868Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2312.json","@type":"CatalogPage","commitId":"204fc820-b27a-4b54-8732-6257b59d7d11","commitTimeStamp":"2017-03-28T09:32:17.6895626Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2313.json","@type":"CatalogPage","commitId":"9f2bb966-6ae0-420e-8f52-5d01e258e586","commitTimeStamp":"2017-03-28T15:03:28.6580003Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2314.json","@type":"CatalogPage","commitId":"4b3987ff-9834-4232-885b-a7c32d2f839e","commitTimeStamp":"2017-03-28T19:43:35.8264162Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2315.json","@type":"CatalogPage","commitId":"ef2852f5-7b1e-45eb-9194-fc87ca4952ec","commitTimeStamp":"2017-03-29T07:31:36.5432645Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2316.json","@type":"CatalogPage","commitId":"eec39434-5ad2-4866-b52c-69d97e9c271d","commitTimeStamp":"2017-03-29T14:16:58.9587318Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2317.json","@type":"CatalogPage","commitId":"ee46fc10-00fd-4875-a78c-5393b773294a","commitTimeStamp":"2017-03-29T20:04:59.5924638Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2318.json","@type":"CatalogPage","commitId":"41ab87ee-1770-44a7-8946-220e4a2f9e0f","commitTimeStamp":"2017-03-30T02:08:30.6215063Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2319.json","@type":"CatalogPage","commitId":"1615a5be-53e5-4fa9-99e0-63ba26a965fe","commitTimeStamp":"2017-03-30T10:39:04.1471794Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2320.json","@type":"CatalogPage","commitId":"9b53cb6f-fcaa-48e7-bff8-b27c9a23d55e","commitTimeStamp":"2017-03-30T15:27:00.4156494Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2321.json","@type":"CatalogPage","commitId":"29316247-7da6-4952-9a61-6266a68fe189","commitTimeStamp":"2017-03-30T17:47:27.5897707Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2322.json","@type":"CatalogPage","commitId":"cd297431-be77-44ec-b249-ae4402c1dd19","commitTimeStamp":"2017-03-31T01:39:16.1504778Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2323.json","@type":"CatalogPage","commitId":"2f2e0988-3748-4ec9-8ba6-a07748a5c752","commitTimeStamp":"2017-03-31T09:56:28.3182781Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2324.json","@type":"CatalogPage","commitId":"e06e3ac8-d92b-4ac9-938c-10af6b606e38","commitTimeStamp":"2017-03-31T15:07:04.4895448Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2325.json","@type":"CatalogPage","commitId":"b4066b89-0607-4ef8-97aa-4473c4d31521","commitTimeStamp":"2017-03-31T21:32:31.3901408Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2326.json","@type":"CatalogPage","commitId":"ab44fea1-4e6d-48a8-9f34-766b627da751","commitTimeStamp":"2017-04-01T05:20:13.7787515Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2327.json","@type":"CatalogPage","commitId":"55b82aec-a072-453b-8422-e47112309fdf","commitTimeStamp":"2017-04-01T16:12:07.4204148Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2328.json","@type":"CatalogPage","commitId":"66c1e64a-5b8d-4dcb-9843-233b4092b56b","commitTimeStamp":"2017-04-02T12:58:45.2782234Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2329.json","@type":"CatalogPage","commitId":"41e9b800-bd0f-4082-b75b-044f6d4601f6","commitTimeStamp":"2017-04-03T04:06:36.8319481Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2330.json","@type":"CatalogPage","commitId":"ce9d80fd-f6e7-4bec-9be4-81f96b7e0edf","commitTimeStamp":"2017-04-03T12:40:09.7358726Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2331.json","@type":"CatalogPage","commitId":"6318b3be-36e9-43b8-b9e0-9076ff3fb5eb","commitTimeStamp":"2017-04-03T18:09:07.3129147Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2332.json","@type":"CatalogPage","commitId":"b6dd317b-f579-42b4-8c9b-8df66d4a1ac7","commitTimeStamp":"2017-04-04T02:53:55.2743894Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2333.json","@type":"CatalogPage","commitId":"1b24b2c1-7347-44f7-9d30-0e412ad697df","commitTimeStamp":"2017-04-04T09:07:59.5371335Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2334.json","@type":"CatalogPage","commitId":"b4f042d0-b256-4f74-8743-10818545b91d","commitTimeStamp":"2017-04-04T16:06:11.4348678Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2335.json","@type":"CatalogPage","commitId":"faa1aba6-f478-402c-82ec-66aa97de801c","commitTimeStamp":"2017-04-05T02:10:09.0063768Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2336.json","@type":"CatalogPage","commitId":"0febf600-ae34-45f2-ac50-c751b0eb3828","commitTimeStamp":"2017-04-05T05:27:42.5785425Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2337.json","@type":"CatalogPage","commitId":"5ac7a8f1-5cc7-4efa-8c34-ddbd7639c8dd","commitTimeStamp":"2017-04-05T09:59:14.4419046Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2338.json","@type":"CatalogPage","commitId":"317261c5-ed4f-4391-be55-2fe7d2c1c3fb","commitTimeStamp":"2017-04-05T14:22:11.7559866Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2339.json","@type":"CatalogPage","commitId":"afaa9b85-a93d-4704-a1e1-363bf53fe674","commitTimeStamp":"2017-04-05T18:47:45.9024685Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2340.json","@type":"CatalogPage","commitId":"d65f2b8e-734b-45e1-ac27-17d2706823f8","commitTimeStamp":"2017-04-06T07:16:15.1008078Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2341.json","@type":"CatalogPage","commitId":"2a46a5b9-9aff-4d7e-8971-f388c5617837","commitTimeStamp":"2017-04-06T10:29:08.9226286Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2342.json","@type":"CatalogPage","commitId":"b178c156-32b7-4f08-86af-4d5c50187452","commitTimeStamp":"2017-04-06T16:22:00.0246662Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2343.json","@type":"CatalogPage","commitId":"37e77ec6-c183-4b77-8ed4-0b5478b66b0b","commitTimeStamp":"2017-04-07T03:22:56.0006991Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2344.json","@type":"CatalogPage","commitId":"9a2cd278-310c-4470-b0d1-d394977ffb74","commitTimeStamp":"2017-04-07T11:08:48.4686621Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2345.json","@type":"CatalogPage","commitId":"0f6c7ef9-b9ad-4174-a744-211181d5103d","commitTimeStamp":"2017-04-07T19:26:16.9192892Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2346.json","@type":"CatalogPage","commitId":"6a72a527-e144-463c-87bd-cb8b3da03d51","commitTimeStamp":"2017-04-07T20:31:57.795706Z","count":533},{"@id":"https://api.nuget.org/v3/catalog0/page2347.json","@type":"CatalogPage","commitId":"fa2bb5ba-203c-4124-86cb-492cbe2a424b","commitTimeStamp":"2017-04-07T20:52:56.249236Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page2348.json","@type":"CatalogPage","commitId":"960fd1b9-b62e-409d-8d86-f1082509a6a0","commitTimeStamp":"2017-04-07T21:04:01.5082757Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2349.json","@type":"CatalogPage","commitId":"dd719d2b-9629-4422-8fee-d6b6b6848ad9","commitTimeStamp":"2017-04-07T21:15:18.3264461Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2350.json","@type":"CatalogPage","commitId":"f1e10352-65b6-4a37-bbcd-7a70b4200f8a","commitTimeStamp":"2017-04-08T07:37:40.4528191Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2351.json","@type":"CatalogPage","commitId":"d642e779-dcad-4621-9ed7-adcf54de8761","commitTimeStamp":"2017-04-08T19:25:29.9309112Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2352.json","@type":"CatalogPage","commitId":"b1b879be-11a1-47ee-ba36-67b8c925bb60","commitTimeStamp":"2017-04-09T11:11:59.0104967Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2353.json","@type":"CatalogPage","commitId":"399a765c-008c-4c5a-9313-0e37b2f5b81f","commitTimeStamp":"2017-04-10T02:59:42.4706118Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2354.json","@type":"CatalogPage","commitId":"aa7e115d-9731-4040-988c-ebdd4a56e838","commitTimeStamp":"2017-04-10T11:05:42.0882022Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2355.json","@type":"CatalogPage","commitId":"54d944bd-20b9-4e8a-bf0f-8e15846c90ef","commitTimeStamp":"2017-04-10T19:10:46.8032263Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2356.json","@type":"CatalogPage","commitId":"800edc9c-fb7d-44fd-bd77-93f7a5154c36","commitTimeStamp":"2017-04-11T06:30:53.4717295Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2357.json","@type":"CatalogPage","commitId":"f365b433-8350-4888-ad25-eaba97a0ec03","commitTimeStamp":"2017-04-11T10:21:28.9300539Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2358.json","@type":"CatalogPage","commitId":"311cb70b-e5f7-4652-a659-c2f599a8103b","commitTimeStamp":"2017-04-11T17:42:36.4085325Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2359.json","@type":"CatalogPage","commitId":"8168df78-15d2-4908-9943-8f7556120521","commitTimeStamp":"2017-04-12T01:30:21.3933656Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2360.json","@type":"CatalogPage","commitId":"b369708a-8513-4c89-b564-162276801fb5","commitTimeStamp":"2017-04-12T08:29:00.516358Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2361.json","@type":"CatalogPage","commitId":"bd098404-aca4-40ef-9673-9b9a296360fe","commitTimeStamp":"2017-04-12T15:10:42.2519629Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2362.json","@type":"CatalogPage","commitId":"19cdb345-36f1-4751-90fd-288c5f1c7537","commitTimeStamp":"2017-04-12T20:20:45.8708141Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2363.json","@type":"CatalogPage","commitId":"b836da14-08ce-4aff-bf42-1308c7e6d5ea","commitTimeStamp":"2017-04-13T07:24:56.2163518Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2364.json","@type":"CatalogPage","commitId":"9b2c7229-dbef-4911-8e03-8e9fe86e03cd","commitTimeStamp":"2017-04-13T15:29:28.5781175Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2365.json","@type":"CatalogPage","commitId":"52cb1922-d5fa-413f-9703-a3ad45189dbb","commitTimeStamp":"2017-04-13T19:44:45.0941652Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2366.json","@type":"CatalogPage","commitId":"14ab8569-db11-4928-ad07-620dc6786ea9","commitTimeStamp":"2017-04-14T10:14:56.1823604Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page2367.json","@type":"CatalogPage","commitId":"fdddb70c-56c6-47a5-9f4c-80a87db45302","commitTimeStamp":"2017-04-14T19:32:17.7625561Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2368.json","@type":"CatalogPage","commitId":"0a31cbdf-ecd8-4ebd-aba7-1dab519d90d3","commitTimeStamp":"2017-04-15T16:04:33.6375447Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2369.json","@type":"CatalogPage","commitId":"2ee8720e-e007-42b6-8e4a-a8d5b4d58b9c","commitTimeStamp":"2017-04-16T07:15:31.6701741Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2370.json","@type":"CatalogPage","commitId":"7b0bf044-8ce0-4ee0-9a2c-79807768e112","commitTimeStamp":"2017-04-17T05:17:25.8775193Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2371.json","@type":"CatalogPage","commitId":"458abdc8-293e-49b1-bdb6-f55d798056e5","commitTimeStamp":"2017-04-17T16:31:00.0838518Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2372.json","@type":"CatalogPage","commitId":"7e85b45f-979d-4c3a-94bc-dec2a1e4169e","commitTimeStamp":"2017-04-18T04:21:14.2626054Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2373.json","@type":"CatalogPage","commitId":"f9599ca1-a99e-4c5b-82b2-6ce0ca0a0343","commitTimeStamp":"2017-04-18T13:29:50.1787455Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2374.json","@type":"CatalogPage","commitId":"aa256fd0-fa89-4d9c-9676-53b8a481acec","commitTimeStamp":"2017-04-18T22:24:22.9072036Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2375.json","@type":"CatalogPage","commitId":"c65434f3-64e5-4b52-a874-ffdfa1719308","commitTimeStamp":"2017-04-19T06:38:00.8925991Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2376.json","@type":"CatalogPage","commitId":"809c4cf7-88df-4cf4-bd9d-264a1ddbbae5","commitTimeStamp":"2017-04-19T11:49:40.326486Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2377.json","@type":"CatalogPage","commitId":"5c989569-80ba-43c6-af78-3a557edba9ad","commitTimeStamp":"2017-04-19T15:44:34.7760152Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2378.json","@type":"CatalogPage","commitId":"622a0d50-d2ec-4a7c-9d47-063f93816db1","commitTimeStamp":"2017-04-19T21:48:05.318859Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2379.json","@type":"CatalogPage","commitId":"1b80c491-27a0-4731-8d41-0aafbadccffb","commitTimeStamp":"2017-04-20T01:16:28.8906375Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2380.json","@type":"CatalogPage","commitId":"dd6c3ff7-d446-47b5-9dd4-c9b4a0b58442","commitTimeStamp":"2017-04-20T05:17:05.1651317Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2381.json","@type":"CatalogPage","commitId":"e2e8fb5e-b2c2-4769-a833-114c87d6f3b2","commitTimeStamp":"2017-04-20T13:04:23.7880133Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2382.json","@type":"CatalogPage","commitId":"34eb3e93-17a7-4641-9137-b1c6501f7246","commitTimeStamp":"2017-04-20T19:26:58.2933816Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2383.json","@type":"CatalogPage","commitId":"918f7a2e-2df0-4f8a-9c01-6ccc9cbf9ac9","commitTimeStamp":"2017-04-21T00:33:53.9720406Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2384.json","@type":"CatalogPage","commitId":"754fd273-d7c9-4dbb-841e-9bd3ada352c6","commitTimeStamp":"2017-04-21T11:07:01.6646748Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2385.json","@type":"CatalogPage","commitId":"c87e6808-1b35-4a9a-bc21-7ffe29c65d31","commitTimeStamp":"2017-04-21T17:17:23.679744Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2386.json","@type":"CatalogPage","commitId":"c1971f98-5332-4621-8866-fa716b8b8849","commitTimeStamp":"2017-04-21T23:50:05.1508118Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2387.json","@type":"CatalogPage","commitId":"2dfd49af-fa8d-4990-9811-cbaf3b6d306c","commitTimeStamp":"2017-04-22T13:47:51.4104358Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2388.json","@type":"CatalogPage","commitId":"35d0be34-030d-4c0b-b450-09a18492b994","commitTimeStamp":"2017-04-23T09:31:28.5945053Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2389.json","@type":"CatalogPage","commitId":"4abbb186-0960-4f28-98c9-97ba0bcfc433","commitTimeStamp":"2017-04-24T03:39:32.9009823Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2390.json","@type":"CatalogPage","commitId":"6fe10585-76fc-49fe-a962-30edf0e3911a","commitTimeStamp":"2017-04-24T11:48:15.2216739Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2391.json","@type":"CatalogPage","commitId":"f3ff645f-e7ff-45d4-8f2e-6b5a58ec9617","commitTimeStamp":"2017-04-24T19:01:31.1564122Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2392.json","@type":"CatalogPage","commitId":"3dafdb93-e8dd-45f2-ad38-6dc3df8ce404","commitTimeStamp":"2017-04-25T04:32:08.3616114Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2393.json","@type":"CatalogPage","commitId":"5829940c-f4a5-4633-9c57-4d98c2769b0b","commitTimeStamp":"2017-04-25T12:43:42.6131301Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2394.json","@type":"CatalogPage","commitId":"aadacf57-9dc6-4aee-8bec-9eea05f46aed","commitTimeStamp":"2017-04-25T20:54:43.3093304Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2395.json","@type":"CatalogPage","commitId":"cdfb3e49-4735-4b29-8826-02e608377a4e","commitTimeStamp":"2017-04-26T09:26:17.9593188Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2396.json","@type":"CatalogPage","commitId":"b413d1a0-145c-4e8a-831f-0c5f617a9bfa","commitTimeStamp":"2017-04-26T14:05:47.0682376Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2397.json","@type":"CatalogPage","commitId":"47d9c700-d353-47ff-bf52-df414b3d816a","commitTimeStamp":"2017-04-26T20:50:19.0958015Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2398.json","@type":"CatalogPage","commitId":"037c0ab2-7b89-4fe9-88d5-5d8c79c2b1a4","commitTimeStamp":"2017-04-27T08:54:45.8155945Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2399.json","@type":"CatalogPage","commitId":"e7092599-63c6-4905-8e62-03df28a3eee9","commitTimeStamp":"2017-04-27T15:22:02.8245949Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2400.json","@type":"CatalogPage","commitId":"e8ca9307-d90b-4838-b773-758f26313271","commitTimeStamp":"2017-04-27T22:38:26.7220337Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2401.json","@type":"CatalogPage","commitId":"0f118e3e-fc73-4a31-aa63-4ec124d1cd40","commitTimeStamp":"2017-04-28T10:21:27.8575474Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2402.json","@type":"CatalogPage","commitId":"69b76ba7-d2ec-4729-8ec6-c63de91052b9","commitTimeStamp":"2017-04-28T15:53:24.0136152Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2403.json","@type":"CatalogPage","commitId":"dbdeb771-2285-47ea-b72f-f5ddb872c4e7","commitTimeStamp":"2017-04-28T22:38:11.5921086Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2404.json","@type":"CatalogPage","commitId":"ede2c081-cf1d-428f-89b8-31de2bdee4bc","commitTimeStamp":"2017-04-29T13:52:50.3525131Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2405.json","@type":"CatalogPage","commitId":"11fdd3f6-806f-40db-9c22-e747fa4d5311","commitTimeStamp":"2017-04-30T11:11:25.8106317Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2406.json","@type":"CatalogPage","commitId":"bda802b3-ac10-4465-aea3-f85147982efe","commitTimeStamp":"2017-05-01T11:11:31.9128791Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2407.json","@type":"CatalogPage","commitId":"33f41502-5092-4932-86e6-81d2d1b48665","commitTimeStamp":"2017-05-01T20:07:56.4772321Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2408.json","@type":"CatalogPage","commitId":"3cb32931-10c8-43eb-9988-589a9f509b1f","commitTimeStamp":"2017-05-02T09:17:07.32454Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2409.json","@type":"CatalogPage","commitId":"55bd192f-5bc4-4ba9-8029-a82b1d570ee7","commitTimeStamp":"2017-05-02T17:05:07.3289609Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2410.json","@type":"CatalogPage","commitId":"181eaace-3c2f-4d3d-8250-e77f08b395bb","commitTimeStamp":"2017-05-03T00:45:17.8843658Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2411.json","@type":"CatalogPage","commitId":"4785b134-bd7a-4773-a244-c29276ba2f6c","commitTimeStamp":"2017-05-03T09:10:53.0951668Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2412.json","@type":"CatalogPage","commitId":"9a16fcba-b86a-4512-b9ed-27cfc7899971","commitTimeStamp":"2017-05-03T16:47:08.9352976Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2413.json","@type":"CatalogPage","commitId":"f4d96be3-7b05-4b8c-91db-89494daffa2b","commitTimeStamp":"2017-05-04T04:48:29.4699903Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2414.json","@type":"CatalogPage","commitId":"37c07fe2-1bf4-44f4-a450-89ab8763694b","commitTimeStamp":"2017-05-04T12:24:07.7173609Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2415.json","@type":"CatalogPage","commitId":"8881f17c-7236-4b75-8db7-58d28aaa2ee8","commitTimeStamp":"2017-05-04T19:39:53.2900872Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2416.json","@type":"CatalogPage","commitId":"6446f955-a7b9-492c-a1cc-b6806c2d2e70","commitTimeStamp":"2017-05-05T02:14:27.6951905Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2417.json","@type":"CatalogPage","commitId":"65b8abec-3fd2-4442-b94e-56ddb69db1b0","commitTimeStamp":"2017-05-05T11:27:19.6319114Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page2418.json","@type":"CatalogPage","commitId":"452bebf9-d90e-41a0-b352-1c8e4d5b2e51","commitTimeStamp":"2017-05-05T17:49:22.7130718Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2419.json","@type":"CatalogPage","commitId":"1c2fe0fb-cf86-4f1b-b76e-8cd24a72ee10","commitTimeStamp":"2017-05-06T08:20:07.0740118Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2420.json","@type":"CatalogPage","commitId":"9ab9bf3b-8601-4889-a9d6-b76fd81ca868","commitTimeStamp":"2017-05-07T03:24:42.704694Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2421.json","@type":"CatalogPage","commitId":"ffbf178a-20cc-45ef-80f0-a767d7f14ff9","commitTimeStamp":"2017-05-08T00:59:46.3473886Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2422.json","@type":"CatalogPage","commitId":"85e7ce26-e5e6-4635-aab2-f57bfa34c67e","commitTimeStamp":"2017-05-08T12:33:01.6326332Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2423.json","@type":"CatalogPage","commitId":"9bb1aacd-0f5a-4a45-9e4c-a151f24ac31d","commitTimeStamp":"2017-05-08T18:32:04.0210962Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2424.json","@type":"CatalogPage","commitId":"da4f4629-384c-4632-b831-e374635beddd","commitTimeStamp":"2017-05-09T02:21:55.7567978Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2425.json","@type":"CatalogPage","commitId":"bf71e08d-df6d-43a9-a29f-b81f3fb0b0b3","commitTimeStamp":"2017-05-09T11:25:38.0358895Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2426.json","@type":"CatalogPage","commitId":"d66c7071-1305-4899-b411-246e3b48004d","commitTimeStamp":"2017-05-09T16:35:09.5958178Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2427.json","@type":"CatalogPage","commitId":"f41ddbba-8069-4fc9-84f8-139102fcba77","commitTimeStamp":"2017-05-09T19:25:35.6463426Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2428.json","@type":"CatalogPage","commitId":"dbab2fbc-a3a3-440d-b197-4c5ae0245157","commitTimeStamp":"2017-05-09T23:16:36.8883655Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2429.json","@type":"CatalogPage","commitId":"03869597-f059-4017-ab4e-59031fef00de","commitTimeStamp":"2017-05-10T08:37:28.331293Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2430.json","@type":"CatalogPage","commitId":"01a547b0-4b88-4059-9f86-df77d0c0b926","commitTimeStamp":"2017-05-10T13:59:47.0143185Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2431.json","@type":"CatalogPage","commitId":"60ab5149-0def-4c66-8142-52dfb42d51d6","commitTimeStamp":"2017-05-10T20:32:21.8140578Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2432.json","@type":"CatalogPage","commitId":"f32115d1-2f08-4976-b08f-59efdcd595ca","commitTimeStamp":"2017-05-11T08:00:44.1381138Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2433.json","@type":"CatalogPage","commitId":"dc7c5965-69d8-4165-a61f-92293d85d27f","commitTimeStamp":"2017-05-11T14:29:25.8299613Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2434.json","@type":"CatalogPage","commitId":"64ec7ad2-bd01-48a8-b520-6eb30ef543ce","commitTimeStamp":"2017-05-11T21:51:48.6354517Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2435.json","@type":"CatalogPage","commitId":"fc40a96a-da65-4808-9f87-f8e3e62c781c","commitTimeStamp":"2017-05-12T05:59:51.9888332Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2436.json","@type":"CatalogPage","commitId":"92390057-88ec-4d3c-85dd-430ffd29d250","commitTimeStamp":"2017-05-12T13:27:22.215145Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2437.json","@type":"CatalogPage","commitId":"0240a935-5437-4d7b-af46-fe35ff136a19","commitTimeStamp":"2017-05-12T19:16:13.4398022Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2438.json","@type":"CatalogPage","commitId":"6e20d439-95e7-4fff-b1e2-27622cb9e737","commitTimeStamp":"2017-05-13T08:42:44.8731147Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2439.json","@type":"CatalogPage","commitId":"aca906b4-0d47-4482-a7ae-109746e66b63","commitTimeStamp":"2017-05-14T01:24:35.2817861Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2440.json","@type":"CatalogPage","commitId":"d976d57b-1834-4f98-9bec-9f89f1ab4925","commitTimeStamp":"2017-05-14T19:17:28.4458994Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2441.json","@type":"CatalogPage","commitId":"69b63b09-33bd-4655-a1a4-10cce85b8153","commitTimeStamp":"2017-05-15T08:47:26.9747296Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2442.json","@type":"CatalogPage","commitId":"3992d287-029b-44e5-9ad0-3949c4cbf48f","commitTimeStamp":"2017-05-15T16:32:52.109789Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2443.json","@type":"CatalogPage","commitId":"6647c5fc-5b31-4892-b5f2-2321eb0a6723","commitTimeStamp":"2017-05-16T03:58:49.8480094Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2444.json","@type":"CatalogPage","commitId":"d535b564-bc8a-46b2-8dfb-04392cd6b5e3","commitTimeStamp":"2017-05-16T12:25:06.8528801Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2445.json","@type":"CatalogPage","commitId":"2aafeb04-8301-41de-af7f-c684bdac5269","commitTimeStamp":"2017-05-16T20:36:57.3468883Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2446.json","@type":"CatalogPage","commitId":"ed98aa46-c04c-4877-97cf-d92b658a6b1d","commitTimeStamp":"2017-05-17T08:34:05.2773282Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2447.json","@type":"CatalogPage","commitId":"7726c5ad-89d7-40aa-9fd7-0c6122af0de6","commitTimeStamp":"2017-05-17T15:46:38.1017051Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2448.json","@type":"CatalogPage","commitId":"cb3ec59b-f5fb-4714-abc5-efff92a8c1c8","commitTimeStamp":"2017-05-17T19:43:25.0289641Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2449.json","@type":"CatalogPage","commitId":"9bf58ebf-421d-46cf-bff9-2de2f353c3b4","commitTimeStamp":"2017-05-18T08:20:53.9491083Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2450.json","@type":"CatalogPage","commitId":"b43eac06-790d-4afb-a939-931f8868ffb6","commitTimeStamp":"2017-05-18T14:24:18.9641482Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2451.json","@type":"CatalogPage","commitId":"592c4dd3-8a43-4c34-9830-32e4b7b42516","commitTimeStamp":"2017-05-18T22:02:36.9234055Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2452.json","@type":"CatalogPage","commitId":"ac46f222-bca5-46be-b5a6-059769fd75f7","commitTimeStamp":"2017-05-19T10:23:51.9202562Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2453.json","@type":"CatalogPage","commitId":"a486d776-76bd-4340-9c0c-3224a80a198e","commitTimeStamp":"2017-05-19T16:48:43.3998317Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2454.json","@type":"CatalogPage","commitId":"9070e277-9cf6-4099-9f12-5004735d4f97","commitTimeStamp":"2017-05-19T22:09:45.8619183Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2455.json","@type":"CatalogPage","commitId":"0ec4162d-5e8d-44e7-84a5-8d4196ee92fb","commitTimeStamp":"2017-05-20T18:33:40.0107505Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2456.json","@type":"CatalogPage","commitId":"be80512a-d1f5-4ded-b276-2e8c3e0bf6f1","commitTimeStamp":"2017-05-21T15:32:46.9986725Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2457.json","@type":"CatalogPage","commitId":"d3b0eee0-f96f-4421-9ca3-e8af9cc5aea1","commitTimeStamp":"2017-05-22T07:52:39.7192271Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2458.json","@type":"CatalogPage","commitId":"b7985019-fea0-4a7f-9b21-e6cb7b9d39bf","commitTimeStamp":"2017-05-22T15:18:28.806417Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2459.json","@type":"CatalogPage","commitId":"75723eba-4a85-44ec-bb72-9636ae4f60c5","commitTimeStamp":"2017-05-22T21:06:35.0165801Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2460.json","@type":"CatalogPage","commitId":"432a4c8a-9ac1-4f87-aac8-0ab535b18ed9","commitTimeStamp":"2017-05-23T06:35:33.4874406Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2461.json","@type":"CatalogPage","commitId":"2e883b71-ebfa-476e-8a43-54daed3efae1","commitTimeStamp":"2017-05-23T13:59:09.5065769Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2462.json","@type":"CatalogPage","commitId":"cb820820-579c-4e63-9d4b-bcb994e8bcc9","commitTimeStamp":"2017-05-23T20:03:48.7795297Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2463.json","@type":"CatalogPage","commitId":"7130a7f1-ba4a-4b19-be89-ce57396e8940","commitTimeStamp":"2017-05-24T05:07:24.0869255Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2464.json","@type":"CatalogPage","commitId":"6d238a88-b88e-4e2e-b0d3-bb1b998fea71","commitTimeStamp":"2017-05-24T09:38:22.4433585Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2465.json","@type":"CatalogPage","commitId":"b4d47bc1-a594-4fc8-aebb-a04f27c20c98","commitTimeStamp":"2017-05-24T15:30:16.5193284Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2466.json","@type":"CatalogPage","commitId":"639f227b-0969-4d76-bb9d-c46407909918","commitTimeStamp":"2017-05-24T23:17:07.7874132Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2467.json","@type":"CatalogPage","commitId":"0f9f5aa3-e378-4a14-999a-90c6e6f668f2","commitTimeStamp":"2017-05-25T10:03:05.1040957Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2468.json","@type":"CatalogPage","commitId":"f5fa9948-0002-46e3-ad4f-bff42395c54d","commitTimeStamp":"2017-05-25T17:03:24.4875089Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2469.json","@type":"CatalogPage","commitId":"56961719-b1ec-4d7a-ad2f-2c1e81ff4e9b","commitTimeStamp":"2017-05-25T22:20:57.3681187Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2470.json","@type":"CatalogPage","commitId":"2de81e37-8d1c-423d-a13e-5219175a6b47","commitTimeStamp":"2017-05-26T09:16:18.6086324Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2471.json","@type":"CatalogPage","commitId":"a6218cb7-c1d1-4904-ac62-2ec4e4e2052a","commitTimeStamp":"2017-05-26T15:22:23.6245882Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2472.json","@type":"CatalogPage","commitId":"d99877e1-b766-4a48-87f6-f456f417d2c7","commitTimeStamp":"2017-05-27T00:55:52.9733157Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2473.json","@type":"CatalogPage","commitId":"46e9c551-fa68-48c5-8ff8-ebd375a5be79","commitTimeStamp":"2017-05-27T21:01:25.4849996Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2474.json","@type":"CatalogPage","commitId":"210f87f2-b4f2-4d3c-8534-5bef0d93b8f6","commitTimeStamp":"2017-05-28T17:48:42.2568351Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2475.json","@type":"CatalogPage","commitId":"007036b1-0dca-4cba-8fcb-398278bd55cb","commitTimeStamp":"2017-05-29T08:38:59.563451Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2476.json","@type":"CatalogPage","commitId":"0a090b45-3662-40d4-a595-5b851ce19b97","commitTimeStamp":"2017-05-29T14:31:18.8726021Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2477.json","@type":"CatalogPage","commitId":"b0b567b4-bcb9-4096-b73c-d0fee51c7017","commitTimeStamp":"2017-05-30T06:01:34.6833274Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2478.json","@type":"CatalogPage","commitId":"f3fd3e85-a5a4-4a4e-b51d-85952cec8fd2","commitTimeStamp":"2017-05-30T13:56:54.9816973Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2479.json","@type":"CatalogPage","commitId":"c86d9fd9-d6aa-4a24-a66d-827560ae0152","commitTimeStamp":"2017-05-30T21:17:31.7182317Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2480.json","@type":"CatalogPage","commitId":"00e910d1-200d-49fd-9df6-5fb23632624d","commitTimeStamp":"2017-05-31T08:35:40.8834612Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2481.json","@type":"CatalogPage","commitId":"559972ec-36da-47aa-b249-ccc502be841b","commitTimeStamp":"2017-05-31T15:03:16.5075226Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2482.json","@type":"CatalogPage","commitId":"62b3ced5-23a4-493f-8dd7-5df6db94e7ca","commitTimeStamp":"2017-05-31T21:20:32.829022Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2483.json","@type":"CatalogPage","commitId":"5ad8cc7d-a462-4ead-9d2a-fa418fbe2af2","commitTimeStamp":"2017-06-01T08:33:29.1085219Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2484.json","@type":"CatalogPage","commitId":"272f4018-9766-40f1-9235-9fa3fb31d411","commitTimeStamp":"2017-06-01T12:00:02.6042542Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2485.json","@type":"CatalogPage","commitId":"3c5571e8-dd16-4300-8ad3-5f60ab981c29","commitTimeStamp":"2017-06-01T13:17:03.1611021Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2486.json","@type":"CatalogPage","commitId":"8e58cdce-a44a-4658-affc-e61517f3b85f","commitTimeStamp":"2017-06-01T17:18:10.865967Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2487.json","@type":"CatalogPage","commitId":"55674358-2c5b-4572-9d9a-95ddc7aa33b0","commitTimeStamp":"2017-06-02T06:35:06.2032159Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2488.json","@type":"CatalogPage","commitId":"536be45d-5a59-4f24-beae-a8e86dc98ce8","commitTimeStamp":"2017-06-02T14:20:33.8837786Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2489.json","@type":"CatalogPage","commitId":"7188fab6-5072-4bfc-9766-33118d12c376","commitTimeStamp":"2017-06-02T21:10:10.3406408Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2490.json","@type":"CatalogPage","commitId":"6be86d6f-1e64-4767-905f-403b137406e7","commitTimeStamp":"2017-06-03T15:38:43.1793354Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2491.json","@type":"CatalogPage","commitId":"986d791c-0d8c-4e41-840d-a94385fec939","commitTimeStamp":"2017-06-04T10:29:29.2497655Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2492.json","@type":"CatalogPage","commitId":"6f12bc72-0a5b-4566-b9a8-6c2ef6886a1a","commitTimeStamp":"2017-06-05T01:00:06.9647565Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2493.json","@type":"CatalogPage","commitId":"20f89ad5-c5a4-46af-be00-aae9cf2d8a3b","commitTimeStamp":"2017-06-05T10:37:00.7528067Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2494.json","@type":"CatalogPage","commitId":"195eb72d-752b-487c-acf7-b1e14b64b17f","commitTimeStamp":"2017-06-05T19:07:50.6318247Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2495.json","@type":"CatalogPage","commitId":"d1acf437-b131-4da6-9a3d-2e399b5b139e","commitTimeStamp":"2017-06-06T08:37:32.2685114Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2496.json","@type":"CatalogPage","commitId":"3edae5a1-4a69-4955-b20f-7a0c1f7c2bc8","commitTimeStamp":"2017-06-06T15:33:51.969518Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2497.json","@type":"CatalogPage","commitId":"19cff9a2-f53f-4bd8-b12f-52e1c2e7ba72","commitTimeStamp":"2017-06-06T20:50:05.0079398Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2498.json","@type":"CatalogPage","commitId":"834bc3cf-73ca-462d-820e-7f258030dbad","commitTimeStamp":"2017-06-07T08:35:07.4945658Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2499.json","@type":"CatalogPage","commitId":"1775f14a-cc0a-4bd8-8b8f-bf31596bd951","commitTimeStamp":"2017-06-07T14:45:44.9007814Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page2500.json","@type":"CatalogPage","commitId":"86b16888-4c4a-44c7-82cb-6ac0f335f011","commitTimeStamp":"2017-06-07T19:09:40.659879Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2501.json","@type":"CatalogPage","commitId":"5cc5658b-bb89-470f-8f84-77a0788ef116","commitTimeStamp":"2017-06-08T01:44:25.9259043Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2502.json","@type":"CatalogPage","commitId":"1c1dd0d7-8c90-4a80-a190-be946774e2eb","commitTimeStamp":"2017-06-08T10:04:05.2598562Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2503.json","@type":"CatalogPage","commitId":"70a7bbcf-64bd-43ed-ad71-27f1190d359f","commitTimeStamp":"2017-06-08T10:56:52.9998592Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2504.json","@type":"CatalogPage","commitId":"8a494d3d-610b-4ed0-aa2e-d153d8ab6596","commitTimeStamp":"2017-06-08T13:46:50.5786043Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2505.json","@type":"CatalogPage","commitId":"be9296dd-8d22-4cc9-bf4e-c4266b482a98","commitTimeStamp":"2017-06-08T19:20:02.0769395Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2506.json","@type":"CatalogPage","commitId":"20f1f06c-3567-4dbb-ba39-630d8c27ce07","commitTimeStamp":"2017-06-09T06:58:46.1952466Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2507.json","@type":"CatalogPage","commitId":"146f323f-59ba-453c-8d7f-f85839886bf6","commitTimeStamp":"2017-06-09T12:41:07.3095069Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2508.json","@type":"CatalogPage","commitId":"f303ce56-1ff9-4c4f-819f-e0362a78eff6","commitTimeStamp":"2017-06-09T21:12:24.7832012Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2509.json","@type":"CatalogPage","commitId":"7ae5d712-1b22-41f4-a098-eb39a58a78ad","commitTimeStamp":"2017-06-10T13:08:39.4061336Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2510.json","@type":"CatalogPage","commitId":"f09634ac-1b79-4367-b96b-fb9b9b217bb5","commitTimeStamp":"2017-06-11T12:56:23.0941518Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page2511.json","@type":"CatalogPage","commitId":"3dccde62-4490-4015-8b8f-ab8f82b569fc","commitTimeStamp":"2017-06-12T01:34:39.9409075Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2512.json","@type":"CatalogPage","commitId":"510b090f-4f6c-4455-889d-decacc487df7","commitTimeStamp":"2017-06-12T14:20:51.3054851Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2513.json","@type":"CatalogPage","commitId":"155885da-1062-4e19-bc8a-19fa26fbd461","commitTimeStamp":"2017-06-13T01:36:12.0087775Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2514.json","@type":"CatalogPage","commitId":"573f7d30-3998-4031-b9dc-0f062c538141","commitTimeStamp":"2017-06-13T09:55:41.6676947Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2515.json","@type":"CatalogPage","commitId":"b6e63d28-c9bb-4b04-8f2d-5eb51bacebcc","commitTimeStamp":"2017-06-13T18:00:45.5258137Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2516.json","@type":"CatalogPage","commitId":"cdc7f0ce-eb35-4775-a985-f8b853bacae6","commitTimeStamp":"2017-06-14T03:35:31.9637345Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2517.json","@type":"CatalogPage","commitId":"471a7998-45d4-4fd2-98ff-c0e6881e617f","commitTimeStamp":"2017-06-14T10:03:24.8823199Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2518.json","@type":"CatalogPage","commitId":"19fb5023-8547-4ce9-8c6e-2ae24a2ea54a","commitTimeStamp":"2017-06-14T16:10:46.6933976Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2519.json","@type":"CatalogPage","commitId":"741ee57c-00ea-4d2b-b212-e49fe1949d2e","commitTimeStamp":"2017-06-15T00:38:38.9020029Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page2520.json","@type":"CatalogPage","commitId":"0ebc985c-8e17-4c54-acd2-61d5a50a1736","commitTimeStamp":"2017-06-15T09:35:51.8714151Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2521.json","@type":"CatalogPage","commitId":"4db24a52-ad18-4ace-9eb5-d736b8b176b0","commitTimeStamp":"2017-06-15T19:45:23.1214619Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2522.json","@type":"CatalogPage","commitId":"25a20860-081d-487e-900a-69bcbe347af0","commitTimeStamp":"2017-06-16T04:08:02.3007682Z","count":534},{"@id":"https://api.nuget.org/v3/catalog0/page2523.json","@type":"CatalogPage","commitId":"b6a0dab4-bb3f-464d-aa01-7d1154549ef4","commitTimeStamp":"2017-06-16T10:28:12.2745047Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2524.json","@type":"CatalogPage","commitId":"e3de2217-f2cf-4072-a700-d18961e509cc","commitTimeStamp":"2017-06-16T20:15:46.4368908Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2525.json","@type":"CatalogPage","commitId":"47d9dad5-55ac-45bd-94e8-1227c16167fb","commitTimeStamp":"2017-06-17T09:24:33.2477243Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2526.json","@type":"CatalogPage","commitId":"e11a8ae7-ca71-48fe-bc49-cae5832313e3","commitTimeStamp":"2017-06-18T05:14:18.9894234Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2527.json","@type":"CatalogPage","commitId":"882e8f11-17c8-470a-8931-3b89b4959e38","commitTimeStamp":"2017-06-18T22:00:46.8684942Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2528.json","@type":"CatalogPage","commitId":"80b611e7-36b3-4581-89a4-41e73eba70db","commitTimeStamp":"2017-06-19T11:16:09.1476211Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2529.json","@type":"CatalogPage","commitId":"f49ab661-31f8-421a-b6b1-7409e06f67c3","commitTimeStamp":"2017-06-19T17:29:21.0345575Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2530.json","@type":"CatalogPage","commitId":"7ca3bb71-3f60-4ed2-a503-a9d91f3c60d1","commitTimeStamp":"2017-06-20T03:51:29.8154703Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2531.json","@type":"CatalogPage","commitId":"3a97be64-822b-41c6-b03b-a2589c265840","commitTimeStamp":"2017-06-20T12:44:59.7127061Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2532.json","@type":"CatalogPage","commitId":"c08e7338-3d37-402c-af2b-964dcee24fb8","commitTimeStamp":"2017-06-20T18:09:04.5561665Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2533.json","@type":"CatalogPage","commitId":"326bf919-c8b0-4628-a97a-d71ccfdd2e75","commitTimeStamp":"2017-06-21T03:17:47.8638971Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page2534.json","@type":"CatalogPage","commitId":"d0845930-d3af-40a9-88f9-627c2ce9143b","commitTimeStamp":"2017-06-21T11:48:40.5009491Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page2535.json","@type":"CatalogPage","commitId":"bc2c3c84-8652-401a-9e73-f80480a40d36","commitTimeStamp":"2017-06-21T19:19:32.7323344Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2536.json","@type":"CatalogPage","commitId":"1e577788-dd03-47f9-b6f2-9be568561609","commitTimeStamp":"2017-06-22T07:19:45.2440036Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2537.json","@type":"CatalogPage","commitId":"67301705-908e-48ff-9f54-bfc1a26fc24c","commitTimeStamp":"2017-06-22T13:55:13.4366018Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2538.json","@type":"CatalogPage","commitId":"358855b5-9d73-4b33-a0f2-63c85aa1f225","commitTimeStamp":"2017-06-22T22:15:35.0422414Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2539.json","@type":"CatalogPage","commitId":"a5333bb9-8804-413a-a8a6-c0e8329f2e35","commitTimeStamp":"2017-06-23T08:58:31.5189659Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2540.json","@type":"CatalogPage","commitId":"0733ada1-ac98-4f92-aa30-da1bcd06a045","commitTimeStamp":"2017-06-23T15:12:50.8008384Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2541.json","@type":"CatalogPage","commitId":"77fd26b8-b8f2-4267-82f8-402167e1b594","commitTimeStamp":"2017-06-23T22:33:39.4436128Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2542.json","@type":"CatalogPage","commitId":"dec0aeb8-8e45-4dcf-8304-91ecafddbea1","commitTimeStamp":"2017-06-24T18:26:13.738195Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2543.json","@type":"CatalogPage","commitId":"2d9c8c40-42fd-43ac-88f9-1100c132014e","commitTimeStamp":"2017-06-25T19:24:58.3590787Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2544.json","@type":"CatalogPage","commitId":"092fc630-b716-4197-b0d0-6ee20a289af4","commitTimeStamp":"2017-06-26T11:26:02.0920102Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2545.json","@type":"CatalogPage","commitId":"735b9d7e-0db4-4aa1-8f2e-7b95e02211d4","commitTimeStamp":"2017-06-26T18:28:04.4558708Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2546.json","@type":"CatalogPage","commitId":"5213a216-55a3-4b42-ad50-34c7be54515f","commitTimeStamp":"2017-06-27T05:09:40.3629263Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2547.json","@type":"CatalogPage","commitId":"a2d49b1e-cef2-4637-abbe-02c332e7210f","commitTimeStamp":"2017-06-27T14:38:23.2692128Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2548.json","@type":"CatalogPage","commitId":"e8e64a59-9f8a-4806-873b-b9113df89c6e","commitTimeStamp":"2017-06-27T23:11:58.2949369Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2549.json","@type":"CatalogPage","commitId":"204866da-31b1-460e-b2cd-83dac9e6da86","commitTimeStamp":"2017-06-28T06:11:47.7848194Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2550.json","@type":"CatalogPage","commitId":"a55738e9-d908-474e-bd6e-ac8164417afc","commitTimeStamp":"2017-06-28T14:22:57.822131Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2551.json","@type":"CatalogPage","commitId":"d65a7a7d-4649-4a02-a78b-f36915545ab2","commitTimeStamp":"2017-06-28T20:07:53.7547188Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2552.json","@type":"CatalogPage","commitId":"2e0816fc-368d-4378-8b0e-80dea816f52c","commitTimeStamp":"2017-06-29T05:44:52.1894914Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2553.json","@type":"CatalogPage","commitId":"fb054d8f-fbb6-4106-8388-c34e76bf609a","commitTimeStamp":"2017-06-29T12:53:57.2765298Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2554.json","@type":"CatalogPage","commitId":"64f48b66-2d6a-4d64-9536-3dee6f0307d8","commitTimeStamp":"2017-06-29T21:08:52.5760599Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2555.json","@type":"CatalogPage","commitId":"1e3fd9cd-25c6-4439-a3a3-2ad6ed788255","commitTimeStamp":"2017-06-30T05:44:41.7100953Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2556.json","@type":"CatalogPage","commitId":"d940aa00-868e-4d25-ad36-541b48fa5aa9","commitTimeStamp":"2017-06-30T14:12:42.0573378Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2557.json","@type":"CatalogPage","commitId":"c8df04f5-49b9-4c6b-a420-cd8e898ed7a0","commitTimeStamp":"2017-06-30T20:41:36.1783381Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2558.json","@type":"CatalogPage","commitId":"a9f91f02-49a8-4759-ae89-bffc511c71e9","commitTimeStamp":"2017-07-01T20:47:27.9947987Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2559.json","@type":"CatalogPage","commitId":"3c631830-ec35-40ef-948d-953e928b303e","commitTimeStamp":"2017-07-02T19:43:20.3468477Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2560.json","@type":"CatalogPage","commitId":"04bb2475-975c-442c-8283-a61be8fa5832","commitTimeStamp":"2017-07-03T10:18:00.0407907Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2561.json","@type":"CatalogPage","commitId":"5600075e-ecda-43da-8b3e-28ef1ae97800","commitTimeStamp":"2017-07-03T17:22:13.3194947Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2562.json","@type":"CatalogPage","commitId":"7ce1559b-3c3a-472d-8526-6eb8fb42b93b","commitTimeStamp":"2017-07-04T02:47:49.8474218Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2563.json","@type":"CatalogPage","commitId":"444240ac-d329-4b1d-8230-54da7eb30231","commitTimeStamp":"2017-07-04T10:45:51.4506998Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2564.json","@type":"CatalogPage","commitId":"9c99d1cd-24b8-44d1-a1fa-55c26186e960","commitTimeStamp":"2017-07-04T18:59:43.1344426Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2565.json","@type":"CatalogPage","commitId":"ee7ab4a7-3f9b-4cbb-b1f5-f5e8b6790375","commitTimeStamp":"2017-07-05T07:12:16.4066793Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2566.json","@type":"CatalogPage","commitId":"1bfcf600-73e4-47f5-9054-8a3a2c47a882","commitTimeStamp":"2017-07-05T13:18:06.7543619Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2567.json","@type":"CatalogPage","commitId":"1cabe5d7-59e9-4a8f-9deb-41655dccbe5c","commitTimeStamp":"2017-07-05T18:52:38.1150368Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2568.json","@type":"CatalogPage","commitId":"53550090-491f-4be0-9bf7-ba8a3dbbb18b","commitTimeStamp":"2017-07-06T00:02:59.9769476Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2569.json","@type":"CatalogPage","commitId":"b2102f90-bda9-439c-9cc4-671f60fb37e6","commitTimeStamp":"2017-07-06T08:00:03.2475995Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2570.json","@type":"CatalogPage","commitId":"8a612b28-aaea-4057-b1b7-137132836696","commitTimeStamp":"2017-07-06T15:30:55.7243076Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2571.json","@type":"CatalogPage","commitId":"7e843e5b-e921-4907-b794-ec7e4613203e","commitTimeStamp":"2017-07-06T21:28:53.7103071Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2572.json","@type":"CatalogPage","commitId":"795c891f-478e-4fb9-ac73-99d45d0d5073","commitTimeStamp":"2017-07-07T07:18:06.8314272Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2573.json","@type":"CatalogPage","commitId":"15be1118-48fa-4de1-9812-98f19a8f5b13","commitTimeStamp":"2017-07-07T12:10:02.2083044Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2574.json","@type":"CatalogPage","commitId":"ddfe340e-871f-4276-b8c8-29d0ff39c117","commitTimeStamp":"2017-07-07T21:07:13.71446Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2575.json","@type":"CatalogPage","commitId":"535e8d4f-7663-4b3d-8ffe-4e64c0f46a82","commitTimeStamp":"2017-07-08T15:16:22.81941Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2576.json","@type":"CatalogPage","commitId":"c7d6fcec-550b-4abe-8a31-e80ea2fdca71","commitTimeStamp":"2017-07-09T17:49:53.6486959Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2577.json","@type":"CatalogPage","commitId":"c6defdde-526d-48c9-8bdd-b275916a098b","commitTimeStamp":"2017-07-10T10:47:31.4217133Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2578.json","@type":"CatalogPage","commitId":"292bf790-6333-4cf2-a915-fd36cc738ba3","commitTimeStamp":"2017-07-10T18:21:14.4014521Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2579.json","@type":"CatalogPage","commitId":"88ebea45-d4d3-4cda-890f-e741b3888083","commitTimeStamp":"2017-07-11T04:00:01.9321646Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2580.json","@type":"CatalogPage","commitId":"05bd2892-6e25-4006-8268-a54811fd3b33","commitTimeStamp":"2017-07-11T13:17:26.388317Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2581.json","@type":"CatalogPage","commitId":"2b28432d-d909-4a33-b909-d9fbcc752bfe","commitTimeStamp":"2017-07-11T20:48:21.9386903Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2582.json","@type":"CatalogPage","commitId":"9fc60de9-2156-46fa-af0e-eb549152157e","commitTimeStamp":"2017-07-12T04:36:43.1529644Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2583.json","@type":"CatalogPage","commitId":"ec236468-ce03-407c-a1a8-93b5515944c9","commitTimeStamp":"2017-07-12T10:58:39.7151584Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2584.json","@type":"CatalogPage","commitId":"73dc8b0e-e553-4efd-8eb6-b3accded2a53","commitTimeStamp":"2017-07-12T18:22:51.6423507Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2585.json","@type":"CatalogPage","commitId":"d30e6108-7904-4544-a2ff-73050f91f148","commitTimeStamp":"2017-07-13T05:58:14.8615333Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2586.json","@type":"CatalogPage","commitId":"b3c3a4c8-750b-4c76-8954-e1d7b9c395a1","commitTimeStamp":"2017-07-13T09:53:13.777133Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2587.json","@type":"CatalogPage","commitId":"ef6658b6-3b84-4306-a6d6-ed86f5c6dc66","commitTimeStamp":"2017-07-13T15:00:23.7282159Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2588.json","@type":"CatalogPage","commitId":"b6683089-790c-433b-ab51-5ee9098e4347","commitTimeStamp":"2017-07-13T21:20:23.6947523Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2589.json","@type":"CatalogPage","commitId":"49ca7596-4bec-48b6-91a5-0eb12a269a49","commitTimeStamp":"2017-07-14T08:29:16.6854807Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2590.json","@type":"CatalogPage","commitId":"f304e398-c352-45c1-8853-f2e89724a45a","commitTimeStamp":"2017-07-14T13:36:43.1251519Z","count":538},{"@id":"https://api.nuget.org/v3/catalog0/page2591.json","@type":"CatalogPage","commitId":"c023cb3a-f32d-48f3-a73e-65d543d4d851","commitTimeStamp":"2017-07-14T18:45:29.4538245Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2592.json","@type":"CatalogPage","commitId":"873b2a31-9236-4e0d-af61-005184449a83","commitTimeStamp":"2017-07-15T05:40:23.1681867Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2593.json","@type":"CatalogPage","commitId":"76a9076f-afb5-486f-beed-398e3e0f93be","commitTimeStamp":"2017-07-15T15:30:07.9708946Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2594.json","@type":"CatalogPage","commitId":"570a0d5b-48f3-48ff-ac7b-cbda7b788297","commitTimeStamp":"2017-07-16T07:40:20.6598242Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2595.json","@type":"CatalogPage","commitId":"56bf8fd0-9d31-4b9d-b863-cc21582d481e","commitTimeStamp":"2017-07-16T21:50:54.0114061Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2596.json","@type":"CatalogPage","commitId":"4c2c0624-8282-487f-8ae2-116cae85a861","commitTimeStamp":"2017-07-17T09:07:37.3777517Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2597.json","@type":"CatalogPage","commitId":"d7a27cb4-bfaa-423f-93d3-b7b1dfdf286b","commitTimeStamp":"2017-07-17T16:33:48.7051289Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2598.json","@type":"CatalogPage","commitId":"e1e9f2d6-44c4-41fa-b2d4-c86838dfbb9f","commitTimeStamp":"2017-07-18T05:52:36.9702175Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2599.json","@type":"CatalogPage","commitId":"2c0f663d-30ae-461f-af49-5c87e42b06a1","commitTimeStamp":"2017-07-18T10:59:11.1251538Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2600.json","@type":"CatalogPage","commitId":"e500d3a9-ee49-499f-84d9-3ec85e826a7d","commitTimeStamp":"2017-07-18T17:45:08.2212112Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2601.json","@type":"CatalogPage","commitId":"3e60e0bc-3907-4ccb-ab37-ecef00ccb293","commitTimeStamp":"2017-07-19T04:44:50.441302Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2602.json","@type":"CatalogPage","commitId":"78f71d2b-d8d9-4db4-8ac5-e771408cca79","commitTimeStamp":"2017-07-19T11:17:36.1541714Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2603.json","@type":"CatalogPage","commitId":"22213f58-43ec-4590-8b7b-03a887aa2c1b","commitTimeStamp":"2017-07-19T16:01:40.3675701Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2604.json","@type":"CatalogPage","commitId":"75b1034f-8361-4278-86a4-68565257b29f","commitTimeStamp":"2017-07-20T05:15:34.957004Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2605.json","@type":"CatalogPage","commitId":"2a750a8f-0a3d-412f-984c-bc19ee9043dd","commitTimeStamp":"2017-07-20T09:36:01.5484454Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2606.json","@type":"CatalogPage","commitId":"8f28cc17-bf01-4a69-8d0f-b289a24acac8","commitTimeStamp":"2017-07-20T18:48:39.9642229Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2607.json","@type":"CatalogPage","commitId":"7d5fc4b8-794f-44e0-a42b-17bcb51bf989","commitTimeStamp":"2017-07-21T05:14:38.2286687Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2608.json","@type":"CatalogPage","commitId":"19bf90dd-4ab4-4169-a61e-665d7bfea09d","commitTimeStamp":"2017-07-21T12:34:19.6735173Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2609.json","@type":"CatalogPage","commitId":"e369104a-f058-419c-a2b4-e146cd6f7ca4","commitTimeStamp":"2017-07-21T16:15:49.9205936Z","count":531},{"@id":"https://api.nuget.org/v3/catalog0/page2610.json","@type":"CatalogPage","commitId":"12960795-a868-409b-a400-bac1e86a0826","commitTimeStamp":"2017-07-21T17:37:00.3863908Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2611.json","@type":"CatalogPage","commitId":"760f4caf-1889-423d-a984-1f43f0284314","commitTimeStamp":"2017-07-22T07:04:15.8263278Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2612.json","@type":"CatalogPage","commitId":"a2388f2c-a26b-425f-947f-0ebb6e272967","commitTimeStamp":"2017-07-22T21:36:40.9370911Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2613.json","@type":"CatalogPage","commitId":"ced1186a-ff2f-4f70-aa7f-4c6517b155b8","commitTimeStamp":"2017-07-23T06:07:05.2633848Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2614.json","@type":"CatalogPage","commitId":"e760bf1c-af90-4e4a-81ae-c7b853ed1cb4","commitTimeStamp":"2017-07-23T07:48:42.8918752Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2615.json","@type":"CatalogPage","commitId":"2526f517-d135-4639-a208-a82a56fd038e","commitTimeStamp":"2017-07-23T09:20:59.2368487Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2616.json","@type":"CatalogPage","commitId":"013e7b9f-5e4b-4929-a182-b60452f329b1","commitTimeStamp":"2017-07-23T11:10:47.3170422Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2617.json","@type":"CatalogPage","commitId":"c06124d7-9874-4e2d-90e8-d52ec292506c","commitTimeStamp":"2017-07-23T11:47:39.137516Z","count":534},{"@id":"https://api.nuget.org/v3/catalog0/page2618.json","@type":"CatalogPage","commitId":"4b997eeb-6dc0-4acc-b2b4-89552d4e46ab","commitTimeStamp":"2017-07-23T13:33:12.5262136Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2619.json","@type":"CatalogPage","commitId":"74690f91-ee04-436e-9037-b26d5492e220","commitTimeStamp":"2017-07-24T07:40:17.4364222Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2620.json","@type":"CatalogPage","commitId":"e17fc6f0-35e3-48a0-95af-2012e1344dbd","commitTimeStamp":"2017-07-24T15:55:49.7547102Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2621.json","@type":"CatalogPage","commitId":"a43095c9-ca25-400c-bbaa-bf554d9d82e4","commitTimeStamp":"2017-07-24T23:38:58.0715358Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2622.json","@type":"CatalogPage","commitId":"17122827-3921-4c8d-8a05-8f114c181ea2","commitTimeStamp":"2017-07-25T08:40:29.0408093Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2623.json","@type":"CatalogPage","commitId":"991464cb-608c-4468-9274-2f304dc9c29e","commitTimeStamp":"2017-07-25T15:57:38.2738939Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2624.json","@type":"CatalogPage","commitId":"3c1f619e-d1ff-4738-b59c-edef66da700c","commitTimeStamp":"2017-07-25T23:46:00.7678009Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2625.json","@type":"CatalogPage","commitId":"8714b276-a1b4-49b5-aa3f-a9e16c7bac5f","commitTimeStamp":"2017-07-26T08:19:54.6293789Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2626.json","@type":"CatalogPage","commitId":"78962726-6f38-4618-8fd8-6b81103d742b","commitTimeStamp":"2017-07-26T11:20:18.2079197Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2627.json","@type":"CatalogPage","commitId":"fd8d250b-6f82-4dcb-a0cf-58fe951c6bd7","commitTimeStamp":"2017-07-26T18:22:21.3626496Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2628.json","@type":"CatalogPage","commitId":"5374dd3d-e7a1-47a1-a80c-e18de48c993c","commitTimeStamp":"2017-07-27T02:49:32.3721841Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2629.json","@type":"CatalogPage","commitId":"875a2a4d-1505-49ed-b8a9-7a5dbd7ff61d","commitTimeStamp":"2017-07-27T10:30:38.266449Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2630.json","@type":"CatalogPage","commitId":"74c9a47f-50b5-4a6c-aca6-7c0c097789c6","commitTimeStamp":"2017-07-27T17:53:46.4165001Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2631.json","@type":"CatalogPage","commitId":"381e99ad-975b-40ef-b54b-3404a6914a90","commitTimeStamp":"2017-07-28T00:02:38.3075899Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2632.json","@type":"CatalogPage","commitId":"087ae5d8-25b8-4f8f-b31e-4ea6923c367c","commitTimeStamp":"2017-07-28T07:23:53.3564817Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2633.json","@type":"CatalogPage","commitId":"dcf19766-d133-4707-9cb7-13c6b81b41b4","commitTimeStamp":"2017-07-28T14:40:41.7543296Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2634.json","@type":"CatalogPage","commitId":"05f93021-222c-4e5f-bcc8-064e44e9b6fa","commitTimeStamp":"2017-07-28T22:19:49.8243121Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2635.json","@type":"CatalogPage","commitId":"e682ed30-39ac-46e6-a800-c242ee828fa1","commitTimeStamp":"2017-07-29T14:58:05.9087057Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2636.json","@type":"CatalogPage","commitId":"eb0e72ca-c94e-49bd-ac95-bec115f29c49","commitTimeStamp":"2017-07-30T14:29:46.2857561Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2637.json","@type":"CatalogPage","commitId":"a22fc47f-5017-462d-be7c-9df5d88b8e74","commitTimeStamp":"2017-07-31T03:02:44.3883244Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2638.json","@type":"CatalogPage","commitId":"08cee9fc-501e-48ea-aecf-9ef6c0761d71","commitTimeStamp":"2017-07-31T10:11:33.14303Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2639.json","@type":"CatalogPage","commitId":"cbcb4215-e703-463c-a71a-70179d5633a5","commitTimeStamp":"2017-07-31T17:27:45.81181Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2640.json","@type":"CatalogPage","commitId":"45c16910-6845-498d-9613-e89811427dd4","commitTimeStamp":"2017-08-01T06:22:15.3494961Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2641.json","@type":"CatalogPage","commitId":"84d7bb68-54fb-44cd-ac8b-fd46f4b36269","commitTimeStamp":"2017-08-01T10:47:34.0915744Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2642.json","@type":"CatalogPage","commitId":"9cf41eb5-d7e6-4902-8cab-9cf517dcf5b1","commitTimeStamp":"2017-08-01T16:31:07.1843154Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2643.json","@type":"CatalogPage","commitId":"278b391c-06bb-4a4d-b540-097287aaab58","commitTimeStamp":"2017-08-01T21:51:15.4479689Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2644.json","@type":"CatalogPage","commitId":"c32ea022-5481-46f4-9fa2-8f60debf9782","commitTimeStamp":"2017-08-02T08:58:16.0062967Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2645.json","@type":"CatalogPage","commitId":"974e43bd-2091-4a37-be13-0f16f8fff7ad","commitTimeStamp":"2017-08-02T13:30:03.7261854Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2646.json","@type":"CatalogPage","commitId":"c7c0e10f-1e6a-4cb4-a3a5-2efb417ec7c7","commitTimeStamp":"2017-08-02T19:36:07.7100677Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2647.json","@type":"CatalogPage","commitId":"0e692e46-fecb-4625-b98a-2b1deba0fddc","commitTimeStamp":"2017-08-03T06:11:14.3120166Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2648.json","@type":"CatalogPage","commitId":"625dc6f0-52e6-45e5-ae0d-6e8988658bb2","commitTimeStamp":"2017-08-03T13:02:46.9334901Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2649.json","@type":"CatalogPage","commitId":"345b5920-9c12-4e23-802b-bd5431010bea","commitTimeStamp":"2017-08-03T20:33:48.8781542Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2650.json","@type":"CatalogPage","commitId":"aa968f46-0359-4888-806d-a6ca4ef7bb10","commitTimeStamp":"2017-08-04T07:30:47.6790053Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2651.json","@type":"CatalogPage","commitId":"827e7d0a-79b4-4731-8b85-b9701eef1d66","commitTimeStamp":"2017-08-04T13:48:16.3027234Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2652.json","@type":"CatalogPage","commitId":"beaaf6be-3db5-4d37-8193-3e5943b4ad15","commitTimeStamp":"2017-08-04T17:17:23.6096501Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2653.json","@type":"CatalogPage","commitId":"77bcba35-a1b6-4b5e-ac68-defdd59a90e4","commitTimeStamp":"2017-08-05T16:21:44.8332849Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2654.json","@type":"CatalogPage","commitId":"4f6e082b-4539-4314-8786-7651bc19d771","commitTimeStamp":"2017-08-06T16:28:23.379398Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2655.json","@type":"CatalogPage","commitId":"f1c31b85-e496-4b54-bdf1-6c5727de5653","commitTimeStamp":"2017-08-07T07:26:05.1176415Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2656.json","@type":"CatalogPage","commitId":"0758eb8e-7117-43eb-bae1-20f31252d75d","commitTimeStamp":"2017-08-07T10:12:34.3736398Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2657.json","@type":"CatalogPage","commitId":"0bf49f9d-f852-4b65-8b59-fd4e6809f6e8","commitTimeStamp":"2017-08-07T16:29:24.1721909Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2658.json","@type":"CatalogPage","commitId":"8f5b0572-91cf-4113-9b61-5f9a648febaf","commitTimeStamp":"2017-08-08T03:24:32.943959Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2659.json","@type":"CatalogPage","commitId":"805f7139-42e1-48e1-b038-5a50e00cce8e","commitTimeStamp":"2017-08-08T13:07:10.7299238Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2660.json","@type":"CatalogPage","commitId":"339426c6-d27f-4c43-be3e-d2a608836486","commitTimeStamp":"2017-08-08T20:41:02.6996845Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2661.json","@type":"CatalogPage","commitId":"dccfe960-81b1-4afc-98a6-a04d942fe6e2","commitTimeStamp":"2017-08-09T08:18:02.493436Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2662.json","@type":"CatalogPage","commitId":"fbe6624f-8d39-4986-80a1-c91e8cf9d2e7","commitTimeStamp":"2017-08-09T14:21:38.3975468Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2663.json","@type":"CatalogPage","commitId":"a90f62ba-1d46-4bb0-a89e-f2ce3f1b0de7","commitTimeStamp":"2017-08-09T19:30:41.6871684Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2664.json","@type":"CatalogPage","commitId":"aa5c1998-f708-4215-b48a-0ca9535f7377","commitTimeStamp":"2017-08-10T06:35:41.7122307Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2665.json","@type":"CatalogPage","commitId":"aae42d14-880c-45aa-978f-8b75b473aff5","commitTimeStamp":"2017-08-10T14:25:12.5536762Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2666.json","@type":"CatalogPage","commitId":"97619e4d-bd94-40fc-a172-70d952427a2a","commitTimeStamp":"2017-08-11T02:18:02.3464271Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2667.json","@type":"CatalogPage","commitId":"277c4a0c-46c9-4cf6-b4c6-18e50d1a9459","commitTimeStamp":"2017-08-11T08:57:49.8177609Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2668.json","@type":"CatalogPage","commitId":"dbaf06cf-2da5-4807-8178-7c1516a7edcc","commitTimeStamp":"2017-08-11T17:58:23.5423712Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2669.json","@type":"CatalogPage","commitId":"a166d780-d9aa-4f34-9d54-6689fb639ca1","commitTimeStamp":"2017-08-11T20:58:06.8854304Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2670.json","@type":"CatalogPage","commitId":"04ac57b7-60c4-4984-9f9b-3ff247601466","commitTimeStamp":"2017-08-12T13:37:06.4333983Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2671.json","@type":"CatalogPage","commitId":"71a15fb1-584b-4096-bf7a-0f0afae8eb3c","commitTimeStamp":"2017-08-13T13:13:34.4439831Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2672.json","@type":"CatalogPage","commitId":"f3e09abe-979b-4e10-a2c2-855f7f782543","commitTimeStamp":"2017-08-14T06:02:55.0359612Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2673.json","@type":"CatalogPage","commitId":"c1e3c4e5-5a42-4c3b-b318-47958ef010ab","commitTimeStamp":"2017-08-14T12:30:08.2839647Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2674.json","@type":"CatalogPage","commitId":"cd9fc206-ef47-4c1f-8a76-bb823562a878","commitTimeStamp":"2017-08-14T19:55:18.8910124Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2675.json","@type":"CatalogPage","commitId":"8534cfb5-8afb-4202-874f-fda941e6c149","commitTimeStamp":"2017-08-15T05:57:34.8747378Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page2676.json","@type":"CatalogPage","commitId":"0aeb5f95-65a6-403c-bc11-185e53247681","commitTimeStamp":"2017-08-15T12:53:23.4351841Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2677.json","@type":"CatalogPage","commitId":"7f78513a-ab6e-4662-862a-c10854633f04","commitTimeStamp":"2017-08-15T16:15:39.0266817Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2678.json","@type":"CatalogPage","commitId":"01f1b011-6eda-4d71-afdd-a56e31d9e634","commitTimeStamp":"2017-08-15T22:13:48.009109Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2679.json","@type":"CatalogPage","commitId":"3f266a9c-d72d-4f6e-bd40-64e0b8f0af83","commitTimeStamp":"2017-08-16T07:31:21.535025Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2680.json","@type":"CatalogPage","commitId":"18ac3335-5308-4f44-a2f1-b10c1eb09ad0","commitTimeStamp":"2017-08-16T12:35:59.6234521Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2681.json","@type":"CatalogPage","commitId":"694e345a-80d3-4703-b9f6-26aad095ac70","commitTimeStamp":"2017-08-16T17:18:21.6043691Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2682.json","@type":"CatalogPage","commitId":"2fb59a69-ed7e-4efd-bd51-40b11775225b","commitTimeStamp":"2017-08-17T00:46:14.4073793Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2683.json","@type":"CatalogPage","commitId":"c7b0ffa7-7b71-4036-a0cb-7d650809b06b","commitTimeStamp":"2017-08-17T08:39:46.4828199Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2684.json","@type":"CatalogPage","commitId":"e9e06e9e-479a-4c4f-8056-3b4024b7bc18","commitTimeStamp":"2017-08-17T16:02:45.863197Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2685.json","@type":"CatalogPage","commitId":"bb44f8ee-4fb6-476b-82fd-b0d6fbdb8b3a","commitTimeStamp":"2017-08-18T03:25:03.0827019Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2686.json","@type":"CatalogPage","commitId":"8302ea8d-6918-4d4b-aace-4096d33eda63","commitTimeStamp":"2017-08-18T09:27:56.1769231Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2687.json","@type":"CatalogPage","commitId":"86261181-daf2-48c8-8f03-54aa14eb22e9","commitTimeStamp":"2017-08-18T15:34:30.5267005Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2688.json","@type":"CatalogPage","commitId":"7d83bc52-2985-4503-b22f-a00c2a9c87de","commitTimeStamp":"2017-08-19T05:43:35.5575178Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2689.json","@type":"CatalogPage","commitId":"1afc86b4-69c2-459b-8bce-f3d336ee662e","commitTimeStamp":"2017-08-19T09:06:10.1565286Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2690.json","@type":"CatalogPage","commitId":"3a4f7d33-6703-4b3b-a167-816e3447aa15","commitTimeStamp":"2017-08-19T10:58:54.3439609Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2691.json","@type":"CatalogPage","commitId":"bb2e792d-3101-4718-bd07-15b97271ca9b","commitTimeStamp":"2017-08-19T12:47:13.2318139Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2692.json","@type":"CatalogPage","commitId":"3efed500-0f38-48fd-9281-c9c8b5ee7eec","commitTimeStamp":"2017-08-19T14:34:15.7888069Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2693.json","@type":"CatalogPage","commitId":"08883a32-149e-4b7e-8259-286ca51a5d1f","commitTimeStamp":"2017-08-19T16:59:00.3058914Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2694.json","@type":"CatalogPage","commitId":"e46830ee-e8fc-4120-b9a4-287d1797f701","commitTimeStamp":"2017-08-20T11:08:28.397938Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2695.json","@type":"CatalogPage","commitId":"72f68514-1315-4524-864e-33bfbbab340f","commitTimeStamp":"2017-08-20T23:21:48.7247833Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2696.json","@type":"CatalogPage","commitId":"b9681649-3b4d-4ed0-bacc-45a5e93eec62","commitTimeStamp":"2017-08-21T08:41:53.4744666Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2697.json","@type":"CatalogPage","commitId":"6a3099a3-c1f4-498e-a8ce-b2df85a81e71","commitTimeStamp":"2017-08-21T15:03:46.2397799Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2698.json","@type":"CatalogPage","commitId":"75b7a9cf-bebf-480a-bd0f-cba5f0d4a740","commitTimeStamp":"2017-08-21T16:02:31.2032541Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2699.json","@type":"CatalogPage","commitId":"7e49ded4-b606-462a-8ecb-4656474d24da","commitTimeStamp":"2017-08-21T16:20:24.0925433Z","count":535},{"@id":"https://api.nuget.org/v3/catalog0/page2700.json","@type":"CatalogPage","commitId":"9f8882f9-daa4-4b00-88aa-b5e0dadc859a","commitTimeStamp":"2017-08-21T16:34:08.0663638Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2701.json","@type":"CatalogPage","commitId":"8799d1c8-e186-4d29-bc4c-32840914fb12","commitTimeStamp":"2017-08-21T17:10:17.3928397Z","count":541},{"@id":"https://api.nuget.org/v3/catalog0/page2702.json","@type":"CatalogPage","commitId":"4188da9b-984b-4491-9494-e55c78081771","commitTimeStamp":"2017-08-21T17:26:26.9476687Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2703.json","@type":"CatalogPage","commitId":"ad7e523b-5068-4e37-a32f-f7b5d510171f","commitTimeStamp":"2017-08-21T17:50:15.1998977Z","count":541},{"@id":"https://api.nuget.org/v3/catalog0/page2704.json","@type":"CatalogPage","commitId":"106ebb6f-1749-4b3e-8d23-f7e1e56a93dc","commitTimeStamp":"2017-08-21T18:06:05.4663021Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2705.json","@type":"CatalogPage","commitId":"6fb00dea-909c-43bd-94f7-433202958ed3","commitTimeStamp":"2017-08-21T18:19:52.4511899Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2706.json","@type":"CatalogPage","commitId":"0aafb3df-a4ed-4e4f-81d9-f37151b6f37e","commitTimeStamp":"2017-08-21T18:34:58.9321989Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2707.json","@type":"CatalogPage","commitId":"6d43e83a-7c1f-4374-9c86-e75e04de2c14","commitTimeStamp":"2017-08-21T18:49:24.8136562Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2708.json","@type":"CatalogPage","commitId":"0d4496be-771b-4d1a-8236-df031d5c0e67","commitTimeStamp":"2017-08-21T19:03:46.5186704Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page2709.json","@type":"CatalogPage","commitId":"55f64d36-4f1f-4fd3-bdcd-e57ff41464c5","commitTimeStamp":"2017-08-22T00:50:40.7044609Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2710.json","@type":"CatalogPage","commitId":"8cb50c46-ce68-45cb-a2f3-a0cb649d007d","commitTimeStamp":"2017-08-22T07:43:22.3458292Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2711.json","@type":"CatalogPage","commitId":"7b31e7ee-25bf-43d5-85a5-092ad450475d","commitTimeStamp":"2017-08-22T11:44:04.8967466Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page2712.json","@type":"CatalogPage","commitId":"e556f904-f592-4e72-82ed-37be06130d45","commitTimeStamp":"2017-08-22T12:03:03.334657Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page2713.json","@type":"CatalogPage","commitId":"1fdfa113-254c-4f3a-9a48-d4bd81fc02ea","commitTimeStamp":"2017-08-22T12:37:14.4119844Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2714.json","@type":"CatalogPage","commitId":"fbe06252-db1a-4a3a-b516-7e0ff2902ee4","commitTimeStamp":"2017-08-22T14:14:05.7786755Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2715.json","@type":"CatalogPage","commitId":"a199eb65-23c9-4a45-b7b1-f9649b9c4a0d","commitTimeStamp":"2017-08-22T22:33:23.120293Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2716.json","@type":"CatalogPage","commitId":"d59f6941-3241-4d0c-8d00-afb2b53eefd7","commitTimeStamp":"2017-08-23T08:43:47.2338578Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2717.json","@type":"CatalogPage","commitId":"2d5f1d40-6887-4072-982f-89bff8489144","commitTimeStamp":"2017-08-23T14:48:01.6959036Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2718.json","@type":"CatalogPage","commitId":"1b939368-d8cf-4140-99f8-88f9bc22b436","commitTimeStamp":"2017-08-23T21:11:10.899057Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2719.json","@type":"CatalogPage","commitId":"d4330edf-7c23-45c7-8b04-8cab8f92bbba","commitTimeStamp":"2017-08-24T07:55:20.4685777Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2720.json","@type":"CatalogPage","commitId":"978f14e2-46a5-4385-943f-150e7cbb401f","commitTimeStamp":"2017-08-24T14:17:23.8474643Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2721.json","@type":"CatalogPage","commitId":"ef821231-fde1-4c54-a1da-800ca7938160","commitTimeStamp":"2017-08-24T19:58:26.4907274Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2722.json","@type":"CatalogPage","commitId":"d5c2a7b2-b0fd-46e4-aa61-2ff4f18cf1e1","commitTimeStamp":"2017-08-25T08:09:08.2382054Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2723.json","@type":"CatalogPage","commitId":"1aefc3f8-475a-4964-a837-c32e596f6719","commitTimeStamp":"2017-08-25T14:53:23.4027168Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2724.json","@type":"CatalogPage","commitId":"1b1f2ca7-923e-457f-8a29-70752c332a46","commitTimeStamp":"2017-08-26T01:12:23.499556Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2725.json","@type":"CatalogPage","commitId":"1d7ccf25-a69d-4cc6-9d88-453daa90583c","commitTimeStamp":"2017-08-26T21:08:25.6704417Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2726.json","@type":"CatalogPage","commitId":"dbcbe59f-73be-4f61-8bd6-393669d08fcb","commitTimeStamp":"2017-08-27T11:54:20.8690039Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2727.json","@type":"CatalogPage","commitId":"01f14923-9782-4907-95cc-07fb1562d436","commitTimeStamp":"2017-08-27T15:59:43.7554034Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2728.json","@type":"CatalogPage","commitId":"ee6dd92e-a4a4-4a8d-b784-25ae0bc6de0e","commitTimeStamp":"2017-08-28T07:56:08.6737768Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2729.json","@type":"CatalogPage","commitId":"5d77f7ef-bb56-49c8-9d0b-070a878c7ed9","commitTimeStamp":"2017-08-28T16:18:15.8597112Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2730.json","@type":"CatalogPage","commitId":"b72d2bb2-0b21-4794-a37f-4e3170db564d","commitTimeStamp":"2017-08-28T21:37:10.7133317Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2731.json","@type":"CatalogPage","commitId":"847216e3-0618-49a2-9143-c4e87543c276","commitTimeStamp":"2017-08-29T08:16:26.8094489Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2732.json","@type":"CatalogPage","commitId":"48dc8961-8679-4f04-a65a-fd5e057efdad","commitTimeStamp":"2017-08-29T13:08:21.0597153Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2733.json","@type":"CatalogPage","commitId":"2d5b2c77-412d-4d44-b051-f647bf3feb2c","commitTimeStamp":"2017-08-29T20:13:12.9252731Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2734.json","@type":"CatalogPage","commitId":"9e56d404-c002-40ae-bd48-abad6507eb4b","commitTimeStamp":"2017-08-30T05:35:06.4307852Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2735.json","@type":"CatalogPage","commitId":"3df5cb1c-88ce-419b-82dd-daf7c43b3ac8","commitTimeStamp":"2017-08-30T12:03:18.9085728Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2736.json","@type":"CatalogPage","commitId":"14d4a73b-f3c5-4687-b7b3-3be85732922a","commitTimeStamp":"2017-08-30T14:02:14.6081827Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2737.json","@type":"CatalogPage","commitId":"751c2fa7-90b5-4e84-a0d3-8009e067ac4d","commitTimeStamp":"2017-08-30T18:14:34.0000917Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2738.json","@type":"CatalogPage","commitId":"b5ec5d21-91fe-4ead-bf7d-ad4062c68684","commitTimeStamp":"2017-08-30T22:59:53.606819Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2739.json","@type":"CatalogPage","commitId":"78b62ec2-83d2-4b76-9e5a-a4ea491369e1","commitTimeStamp":"2017-08-31T07:12:44.6417772Z","count":541},{"@id":"https://api.nuget.org/v3/catalog0/page2740.json","@type":"CatalogPage","commitId":"51a096f9-e934-484f-b576-df0ef0303dd2","commitTimeStamp":"2017-08-31T13:01:11.2513139Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2741.json","@type":"CatalogPage","commitId":"f66f0cee-32a9-4904-8bd4-8b9afc855c63","commitTimeStamp":"2017-08-31T19:30:24.7638595Z","count":536},{"@id":"https://api.nuget.org/v3/catalog0/page2742.json","@type":"CatalogPage","commitId":"f14e7734-56de-4041-91d6-6195d5001196","commitTimeStamp":"2017-09-01T07:02:26.704155Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2743.json","@type":"CatalogPage","commitId":"0a26133c-b81e-4fa3-9ce1-b45b4272260b","commitTimeStamp":"2017-09-01T10:01:46.6804896Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2744.json","@type":"CatalogPage","commitId":"49a3ed07-d4ae-45c6-9090-dbdf037818fd","commitTimeStamp":"2017-09-01T14:52:15.2600425Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2745.json","@type":"CatalogPage","commitId":"8af5a315-c766-44bf-82b7-c98682ac5584","commitTimeStamp":"2017-09-02T03:17:18.9756612Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2746.json","@type":"CatalogPage","commitId":"a4328f2a-d0f9-45d8-ba56-495b1c252d41","commitTimeStamp":"2017-09-02T22:38:25.6259565Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2747.json","@type":"CatalogPage","commitId":"9b0ec374-1cb4-4247-84c7-d6087ffd1981","commitTimeStamp":"2017-09-03T19:21:01.1095955Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2748.json","@type":"CatalogPage","commitId":"9530505a-4e90-4cec-a3e7-eeee056fc94b","commitTimeStamp":"2017-09-04T08:51:56.9371065Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2749.json","@type":"CatalogPage","commitId":"3d3ed194-030a-4fa3-87dd-17e42e079f96","commitTimeStamp":"2017-09-04T15:20:12.7254112Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2750.json","@type":"CatalogPage","commitId":"3820ab7c-b62f-464a-b065-b3b5ce80eb6b","commitTimeStamp":"2017-09-04T20:50:16.1309936Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2751.json","@type":"CatalogPage","commitId":"5587968c-f399-4790-a5c1-014657e79324","commitTimeStamp":"2017-09-05T06:11:46.4689918Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2752.json","@type":"CatalogPage","commitId":"54240501-e10c-4409-8369-d69ca91cfb48","commitTimeStamp":"2017-09-05T11:54:23.7643979Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2753.json","@type":"CatalogPage","commitId":"7d4685c0-34de-4574-82eb-3c6a866cd83e","commitTimeStamp":"2017-09-05T16:30:48.1949774Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2754.json","@type":"CatalogPage","commitId":"f982d849-5d1d-48e9-88f8-d4213ade07bd","commitTimeStamp":"2017-09-06T00:56:07.3773317Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2755.json","@type":"CatalogPage","commitId":"12993951-78b8-428c-854a-8c6a9bc511b4","commitTimeStamp":"2017-09-06T07:10:58.6527142Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page2756.json","@type":"CatalogPage","commitId":"2ca80fec-d143-466b-868e-63ae62aa5058","commitTimeStamp":"2017-09-06T12:24:47.4390742Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2757.json","@type":"CatalogPage","commitId":"4b902635-f840-44c4-97b4-78f7740f5616","commitTimeStamp":"2017-09-06T17:27:27.6926763Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2758.json","@type":"CatalogPage","commitId":"89231ea6-dd99-4c47-af4f-27147275a785","commitTimeStamp":"2017-09-06T23:30:38.1337179Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2759.json","@type":"CatalogPage","commitId":"674cd491-5a84-4197-b45a-d478b3640730","commitTimeStamp":"2017-09-07T06:33:07.182553Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2760.json","@type":"CatalogPage","commitId":"2aa5c910-e855-40f9-9fe5-9a63fcb124d3","commitTimeStamp":"2017-09-07T12:57:54.5471965Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2761.json","@type":"CatalogPage","commitId":"975dc65e-3a25-4370-9fdb-5cbd93b78aae","commitTimeStamp":"2017-09-07T18:44:40.9685755Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2762.json","@type":"CatalogPage","commitId":"e677c9c6-dd44-41db-84e4-2272d707a96e","commitTimeStamp":"2017-09-08T04:23:21.9257069Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2763.json","@type":"CatalogPage","commitId":"3856b6bf-5c4a-4b27-9d3d-2850d6ac2b2e","commitTimeStamp":"2017-09-08T09:47:57.6992295Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2764.json","@type":"CatalogPage","commitId":"f4970313-ece8-4df9-8119-c85a05689eaf","commitTimeStamp":"2017-09-08T13:41:42.8050141Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2765.json","@type":"CatalogPage","commitId":"ff82e6f1-0bdb-48fb-9ddd-2629baefd8ca","commitTimeStamp":"2017-09-08T20:43:38.1043191Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2766.json","@type":"CatalogPage","commitId":"41fb396e-9f91-421f-a86a-081c47f7255d","commitTimeStamp":"2017-09-08T22:39:17.476813Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2767.json","@type":"CatalogPage","commitId":"48e8e091-7681-43a8-b438-05089fb60ddc","commitTimeStamp":"2017-09-09T07:25:20.475249Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2768.json","@type":"CatalogPage","commitId":"e8dc582d-9573-4e7c-b802-675fe6020920","commitTimeStamp":"2017-09-10T00:59:20.2175089Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2769.json","@type":"CatalogPage","commitId":"c020c49c-a8e7-4ab1-ad3d-214ef3fccc45","commitTimeStamp":"2017-09-10T14:09:56.1678402Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2770.json","@type":"CatalogPage","commitId":"c11c951f-f217-423e-a374-5e096987d175","commitTimeStamp":"2017-09-11T07:08:42.9477939Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2771.json","@type":"CatalogPage","commitId":"6bb973a4-e139-454f-bb99-a84e6fa484e0","commitTimeStamp":"2017-09-11T13:57:27.0847405Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2772.json","@type":"CatalogPage","commitId":"ae99184a-725b-4515-924d-24e4c6c47924","commitTimeStamp":"2017-09-11T19:24:33.7127647Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2773.json","@type":"CatalogPage","commitId":"09486409-a28a-458a-87d4-c0769a32e313","commitTimeStamp":"2017-09-12T06:44:48.3483395Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2774.json","@type":"CatalogPage","commitId":"ff4ac1f3-5f4a-451f-9099-b23da2d7a1a5","commitTimeStamp":"2017-09-12T11:01:06.6730188Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2775.json","@type":"CatalogPage","commitId":"c3d78ea4-b398-4d69-8b7a-58470a54362c","commitTimeStamp":"2017-09-12T15:24:33.9388757Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2776.json","@type":"CatalogPage","commitId":"847fa02b-1b6f-43ce-bd1e-e5437ffb5854","commitTimeStamp":"2017-09-12T20:19:15.0007319Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2777.json","@type":"CatalogPage","commitId":"d173af9e-18b2-446d-bb35-cbddf0aca03a","commitTimeStamp":"2017-09-13T07:18:11.0087513Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2778.json","@type":"CatalogPage","commitId":"f5a2c465-fd5a-46fa-ad44-099564cad228","commitTimeStamp":"2017-09-13T08:33:19.7924018Z","count":531},{"@id":"https://api.nuget.org/v3/catalog0/page2779.json","@type":"CatalogPage","commitId":"28b6c507-d672-42b2-ae39-5e2648773b88","commitTimeStamp":"2017-09-13T09:58:07.4227323Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2780.json","@type":"CatalogPage","commitId":"3403a8eb-4412-4a39-981d-517fcbe3ea80","commitTimeStamp":"2017-09-13T11:16:56.4202506Z","count":541},{"@id":"https://api.nuget.org/v3/catalog0/page2781.json","@type":"CatalogPage","commitId":"93ebefbb-d0b0-4259-bb1f-c171ddbb0b7e","commitTimeStamp":"2017-09-13T12:44:58.6264616Z","count":541},{"@id":"https://api.nuget.org/v3/catalog0/page2782.json","@type":"CatalogPage","commitId":"d19ca6ce-5998-47d9-b629-cdc361910b6a","commitTimeStamp":"2017-09-13T14:31:44.8100879Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2783.json","@type":"CatalogPage","commitId":"eda73312-f063-41b8-8131-eb2d211deac3","commitTimeStamp":"2017-09-13T17:08:08.0442651Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page2784.json","@type":"CatalogPage","commitId":"47ceb9cd-a95f-4d6f-a27a-b678a43450ce","commitTimeStamp":"2017-09-14T01:35:38.263954Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2785.json","@type":"CatalogPage","commitId":"79e36769-1d07-4b11-94f0-29e1df36f4ce","commitTimeStamp":"2017-09-14T11:26:34.8588115Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2786.json","@type":"CatalogPage","commitId":"9bd436e8-92b7-4e51-ab53-dad83ff56682","commitTimeStamp":"2017-09-14T18:43:14.1908333Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2787.json","@type":"CatalogPage","commitId":"dcfc1456-6691-442e-93ab-5be1c3e982b8","commitTimeStamp":"2017-09-15T00:38:04.2778064Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2788.json","@type":"CatalogPage","commitId":"265f08f5-4df7-40c2-8438-617f6e06102a","commitTimeStamp":"2017-09-15T12:32:54.2678097Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2789.json","@type":"CatalogPage","commitId":"b7643efe-a1c6-49ed-a81e-b20edec6cf5c","commitTimeStamp":"2017-09-15T19:34:16.9793018Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2790.json","@type":"CatalogPage","commitId":"e7c8d5de-c5ef-4fce-9c2c-0e52bf3eeda8","commitTimeStamp":"2017-09-16T09:57:55.474857Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2791.json","@type":"CatalogPage","commitId":"8e97d861-410c-47bc-9da8-bb909d351408","commitTimeStamp":"2017-09-17T06:52:48.0073433Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2792.json","@type":"CatalogPage","commitId":"1bc1855b-7327-4718-9e31-650745a099d6","commitTimeStamp":"2017-09-18T00:17:46.9719019Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2793.json","@type":"CatalogPage","commitId":"0e26a256-4866-4e75-a92d-43fa99f380b9","commitTimeStamp":"2017-09-18T10:43:39.7529309Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2794.json","@type":"CatalogPage","commitId":"5cae7182-d201-4f97-bba9-64bbdaf7d5c4","commitTimeStamp":"2017-09-18T15:22:06.7707397Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2795.json","@type":"CatalogPage","commitId":"e2a076b7-be68-45a4-a0d9-7f88c34e5d11","commitTimeStamp":"2017-09-18T18:38:09.1634065Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2796.json","@type":"CatalogPage","commitId":"aa403788-dbec-47e5-b952-4d49d4422479","commitTimeStamp":"2017-09-19T00:01:55.2490635Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2797.json","@type":"CatalogPage","commitId":"4c47e9c4-7699-4d8a-9851-5e56f3fe4909","commitTimeStamp":"2017-09-19T08:55:52.558155Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2798.json","@type":"CatalogPage","commitId":"a0552057-44d1-4e3b-a958-75f63607a934","commitTimeStamp":"2017-09-19T12:59:24.7963939Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2799.json","@type":"CatalogPage","commitId":"4f3e0443-c7bc-468c-a94c-a1c4412c7280","commitTimeStamp":"2017-09-19T18:18:20.3269674Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2800.json","@type":"CatalogPage","commitId":"c67d0c2f-8ca3-4b16-9aed-5b5b8f8191ab","commitTimeStamp":"2017-09-20T04:15:02.3119643Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2801.json","@type":"CatalogPage","commitId":"5d1935b2-15ca-4515-a9f5-b0b384dd3832","commitTimeStamp":"2017-09-20T08:37:44.3908068Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2802.json","@type":"CatalogPage","commitId":"77f6b2ee-333d-4bf0-9a2b-c2b09e1edf23","commitTimeStamp":"2017-09-20T15:42:37.873333Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2803.json","@type":"CatalogPage","commitId":"a2b34baa-de89-4a62-8455-ee00b3845a28","commitTimeStamp":"2017-09-20T17:01:19.6568174Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page2804.json","@type":"CatalogPage","commitId":"128f2bdf-27ef-411d-8e86-dd9d96aea98b","commitTimeStamp":"2017-09-20T20:21:40.3268953Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2805.json","@type":"CatalogPage","commitId":"b9d2dcad-f800-4c06-a73f-d02426cbf5c2","commitTimeStamp":"2017-09-21T05:33:37.9614579Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2806.json","@type":"CatalogPage","commitId":"dd7f98b0-20d7-4d9a-b12d-a5a3f0456cf9","commitTimeStamp":"2017-09-21T09:02:07.2988947Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2807.json","@type":"CatalogPage","commitId":"c27d311b-75b1-40c6-9a53-dd8d5b201553","commitTimeStamp":"2017-09-21T15:15:45.479387Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2808.json","@type":"CatalogPage","commitId":"df62b208-8a8d-4968-b628-0ec9dc8c4f4a","commitTimeStamp":"2017-09-21T23:29:08.1730118Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2809.json","@type":"CatalogPage","commitId":"69d3b2d3-e70c-48f7-be98-01b8ef24c018","commitTimeStamp":"2017-09-22T07:16:03.7972105Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2810.json","@type":"CatalogPage","commitId":"ec878889-ecc0-4cd6-878c-757f7cee98cb","commitTimeStamp":"2017-09-22T12:20:26.8742858Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2811.json","@type":"CatalogPage","commitId":"68143ed4-c7dc-48a9-97cd-1b8aa48db6d0","commitTimeStamp":"2017-09-22T18:53:26.6490491Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2812.json","@type":"CatalogPage","commitId":"5a3bdd06-8765-499f-a5ee-2a1d2e8b145e","commitTimeStamp":"2017-09-23T08:22:31.1056776Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2813.json","@type":"CatalogPage","commitId":"d73a10a8-075b-42c6-bbe7-0f5ba46ff333","commitTimeStamp":"2017-09-24T02:33:03.1246098Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2814.json","@type":"CatalogPage","commitId":"74ada193-ef29-48d2-84f5-1b614b8e0404","commitTimeStamp":"2017-09-24T21:50:28.7537208Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2815.json","@type":"CatalogPage","commitId":"b45cfb91-309c-4ea1-b9de-89624b4b1aa2","commitTimeStamp":"2017-09-25T01:55:19.7682325Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page2816.json","@type":"CatalogPage","commitId":"2e124373-6df1-4b03-bc05-ca6818df1e57","commitTimeStamp":"2017-09-25T03:40:26.6758845Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2817.json","@type":"CatalogPage","commitId":"14affd79-ad83-4359-8490-1b911be9edb5","commitTimeStamp":"2017-09-25T05:32:24.2664576Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2818.json","@type":"CatalogPage","commitId":"7f6efae1-bfe3-4307-9b2b-05af95f7bebf","commitTimeStamp":"2017-09-25T07:16:42.1689805Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2819.json","@type":"CatalogPage","commitId":"cad8c21e-e809-4830-9462-372567eea653","commitTimeStamp":"2017-09-25T09:32:22.4106919Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2820.json","@type":"CatalogPage","commitId":"f017a92e-9282-4a56-8369-b50b30a8bf8e","commitTimeStamp":"2017-09-25T15:38:19.196076Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2821.json","@type":"CatalogPage","commitId":"294981f0-747d-4abb-a7cc-594e3e4d89fc","commitTimeStamp":"2017-09-25T21:29:21.0477409Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page2822.json","@type":"CatalogPage","commitId":"071c2b09-105f-469a-950f-aab0a54e9110","commitTimeStamp":"2017-09-26T07:08:36.3544262Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2823.json","@type":"CatalogPage","commitId":"d6526a20-649e-4fed-af9a-fe502bec8e38","commitTimeStamp":"2017-09-26T11:59:46.154692Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2824.json","@type":"CatalogPage","commitId":"e89b39ca-10ef-47c8-9569-93342289cef1","commitTimeStamp":"2017-09-26T17:51:48.8938858Z","count":541},{"@id":"https://api.nuget.org/v3/catalog0/page2825.json","@type":"CatalogPage","commitId":"b770a439-786e-4a00-bc1b-3701bfc24173","commitTimeStamp":"2017-09-27T02:13:58.319864Z","count":541},{"@id":"https://api.nuget.org/v3/catalog0/page2826.json","@type":"CatalogPage","commitId":"a1ef9679-36aa-4669-aab6-5f7253fb6a1b","commitTimeStamp":"2017-09-27T11:35:12.3264517Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2827.json","@type":"CatalogPage","commitId":"4092eb9d-bed1-4618-9f7a-e717c3407e04","commitTimeStamp":"2017-09-27T15:27:36.9496135Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2828.json","@type":"CatalogPage","commitId":"e06b8e2e-683a-4dc9-a7ca-88410977dc6a","commitTimeStamp":"2017-09-27T23:41:02.4534321Z","count":536},{"@id":"https://api.nuget.org/v3/catalog0/page2829.json","@type":"CatalogPage","commitId":"673697e5-6e26-4e15-99de-051cae2176e7","commitTimeStamp":"2017-09-28T10:39:58.0818909Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2830.json","@type":"CatalogPage","commitId":"67db116c-4f25-4dc1-86b1-a74334d76af4","commitTimeStamp":"2017-09-28T17:36:13.1688267Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2831.json","@type":"CatalogPage","commitId":"e066fcd5-27ad-4614-93a5-39c4f99ec857","commitTimeStamp":"2017-09-29T03:55:32.4428146Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2832.json","@type":"CatalogPage","commitId":"05b3356e-b758-4dcc-a2a4-759cb291a69d","commitTimeStamp":"2017-09-29T12:48:22.4214061Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2833.json","@type":"CatalogPage","commitId":"275fad36-c35b-4e8a-9b6e-1d224d2c658a","commitTimeStamp":"2017-09-29T20:27:23.7548241Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2834.json","@type":"CatalogPage","commitId":"beb4b7b6-711d-46a2-9268-223405c1dde9","commitTimeStamp":"2017-09-30T14:03:05.6347662Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2835.json","@type":"CatalogPage","commitId":"5f98d66b-beab-4757-b3dc-89bb8cd10329","commitTimeStamp":"2017-10-01T12:30:32.9621669Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2836.json","@type":"CatalogPage","commitId":"f50f2ad5-dd56-440e-b122-a2327255e36d","commitTimeStamp":"2017-10-01T22:05:37.6729553Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2837.json","@type":"CatalogPage","commitId":"6dbb244a-040d-4390-9f27-4d4c94a7e7af","commitTimeStamp":"2017-10-02T04:38:18.923159Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2838.json","@type":"CatalogPage","commitId":"676953ec-642f-4d83-8179-5fe0544ccd0b","commitTimeStamp":"2017-10-02T12:49:29.4081055Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2839.json","@type":"CatalogPage","commitId":"0e6f8b6d-a1b4-456f-8ace-a9f4282a8730","commitTimeStamp":"2017-10-02T17:31:45.6153636Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2840.json","@type":"CatalogPage","commitId":"851c853c-a4bc-489b-9196-fca2e4b007ec","commitTimeStamp":"2017-10-03T02:50:08.3179412Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2841.json","@type":"CatalogPage","commitId":"2de35907-ec15-4fb7-914f-30e6c09e0651","commitTimeStamp":"2017-10-03T09:25:37.945236Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2842.json","@type":"CatalogPage","commitId":"b4a4c4e1-e8b8-4219-b253-4722236375e3","commitTimeStamp":"2017-10-03T14:41:11.9050932Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2843.json","@type":"CatalogPage","commitId":"4690cec8-5464-4dee-b9c7-e1d726437dc0","commitTimeStamp":"2017-10-03T20:33:19.3904452Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2844.json","@type":"CatalogPage","commitId":"a2c0cb5b-a0c1-4fe1-9afe-82c6bb99a5b5","commitTimeStamp":"2017-10-04T01:20:47.9971568Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2845.json","@type":"CatalogPage","commitId":"6f8d713a-becf-436d-9184-73a3023182fd","commitTimeStamp":"2017-10-04T08:08:46.7307347Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2846.json","@type":"CatalogPage","commitId":"448405f0-5a97-46c0-8a9c-b8ffa957ab77","commitTimeStamp":"2017-10-04T14:38:43.7513368Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2847.json","@type":"CatalogPage","commitId":"ce7c3cfa-ddfa-4662-804b-45526a76aa76","commitTimeStamp":"2017-10-04T18:28:03.054163Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2848.json","@type":"CatalogPage","commitId":"2bcde4ee-5a19-43e0-8104-340fcd335b20","commitTimeStamp":"2017-10-05T00:58:03.2239527Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page2849.json","@type":"CatalogPage","commitId":"4f54a5fe-90b7-499a-8e3e-28ed5ac6eb9d","commitTimeStamp":"2017-10-05T08:32:39.6727744Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2850.json","@type":"CatalogPage","commitId":"c2af4348-e5f3-4c0d-8927-d4a6800cb40c","commitTimeStamp":"2017-10-05T15:24:34.632818Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2851.json","@type":"CatalogPage","commitId":"ca5fd6fe-cf4b-4cf1-9b28-f30ba7bb7910","commitTimeStamp":"2017-10-05T23:31:44.5199142Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2852.json","@type":"CatalogPage","commitId":"94101a07-0c99-48cd-ab47-bb54334120af","commitTimeStamp":"2017-10-06T09:01:13.4788131Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2853.json","@type":"CatalogPage","commitId":"8d4766b4-6c44-4e76-885f-94536deb5b0a","commitTimeStamp":"2017-10-06T14:56:42.6192901Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2854.json","@type":"CatalogPage","commitId":"aa2f6e10-4a66-4622-9c46-4d85b05d8b4e","commitTimeStamp":"2017-10-06T20:21:02.9639998Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2855.json","@type":"CatalogPage","commitId":"8d20c96c-e9fa-448a-a04c-cb6db3207ed3","commitTimeStamp":"2017-10-07T09:30:38.169338Z","count":539},{"@id":"https://api.nuget.org/v3/catalog0/page2856.json","@type":"CatalogPage","commitId":"fa482e38-2bb5-4b1c-b666-4fde443e7f98","commitTimeStamp":"2017-10-07T22:21:26.0885373Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2857.json","@type":"CatalogPage","commitId":"60f6e2ff-a868-4155-8705-0b316f9f436f","commitTimeStamp":"2017-10-08T15:15:42.319325Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2858.json","@type":"CatalogPage","commitId":"fb15f5bd-fa7e-453e-ae8a-0ca3f655b406","commitTimeStamp":"2017-10-09T06:13:21.923978Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2859.json","@type":"CatalogPage","commitId":"49a2cc98-11e0-495b-8dd3-6668af04ee93","commitTimeStamp":"2017-10-09T13:21:57.8268162Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2860.json","@type":"CatalogPage","commitId":"362e8e60-c13c-4035-9b60-9bd687d38470","commitTimeStamp":"2017-10-09T19:11:33.0958453Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2861.json","@type":"CatalogPage","commitId":"fb7242a4-02e8-4229-9329-5b086b1248a4","commitTimeStamp":"2017-10-10T07:11:00.5746156Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2862.json","@type":"CatalogPage","commitId":"2a40631b-743e-4790-bf51-20ba5a567d46","commitTimeStamp":"2017-10-10T13:24:52.7470596Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2863.json","@type":"CatalogPage","commitId":"02a3272a-4467-40ec-a716-6d90a12ce92e","commitTimeStamp":"2017-10-10T18:14:55.8199019Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2864.json","@type":"CatalogPage","commitId":"39257a59-a94d-4cff-803f-4bd5b773a193","commitTimeStamp":"2017-10-11T00:51:02.4765391Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2865.json","@type":"CatalogPage","commitId":"40d1f0bc-5e59-4efe-9190-b305d6e76c42","commitTimeStamp":"2017-10-11T08:46:10.1786927Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2866.json","@type":"CatalogPage","commitId":"c7cdc44f-36ea-46ab-a9ac-28297e4cf548","commitTimeStamp":"2017-10-11T14:35:20.8798762Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2867.json","@type":"CatalogPage","commitId":"21aacb98-8043-447d-95e7-b7b71d2c4fc5","commitTimeStamp":"2017-10-11T20:33:12.2621679Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2868.json","@type":"CatalogPage","commitId":"2f45fb29-6e20-471c-a49f-604fb1b8e8aa","commitTimeStamp":"2017-10-12T10:19:18.6288779Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2869.json","@type":"CatalogPage","commitId":"79efc0e3-e774-46f7-81bd-bb0f465f2d07","commitTimeStamp":"2017-10-12T16:47:13.8214956Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2870.json","@type":"CatalogPage","commitId":"44f41c8c-4046-43ef-9bd4-400339031dac","commitTimeStamp":"2017-10-12T20:49:42.561161Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2871.json","@type":"CatalogPage","commitId":"99f77248-da21-43f5-a6d4-7fa68888e356","commitTimeStamp":"2017-10-13T05:55:22.831741Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2872.json","@type":"CatalogPage","commitId":"f7d1fbbc-58e0-46a8-bd69-e7891b312957","commitTimeStamp":"2017-10-13T11:14:22.3510522Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page2873.json","@type":"CatalogPage","commitId":"4a3e8ab7-5591-4fac-ab78-abddbfd30cfb","commitTimeStamp":"2017-10-13T14:54:20.5715162Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2874.json","@type":"CatalogPage","commitId":"34f6089e-3f25-4237-8761-5944ce0db56c","commitTimeStamp":"2017-10-13T22:55:21.2179505Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page2875.json","@type":"CatalogPage","commitId":"87ecbc7e-64ee-4910-873e-159772c6c1c3","commitTimeStamp":"2017-10-14T17:40:06.4388966Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2876.json","@type":"CatalogPage","commitId":"d499b29a-c492-4cfb-8139-db33e10bd1c5","commitTimeStamp":"2017-10-15T18:12:17.2610952Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2877.json","@type":"CatalogPage","commitId":"e2f23589-1e23-4022-b76d-838627b73fa8","commitTimeStamp":"2017-10-16T05:16:10.118176Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2878.json","@type":"CatalogPage","commitId":"2b4fc091-1780-47e7-8d00-cab8da49b44e","commitTimeStamp":"2017-10-16T11:46:42.1279304Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2879.json","@type":"CatalogPage","commitId":"f6c3644b-21bb-4d94-aa38-d61f19538bfa","commitTimeStamp":"2017-10-16T18:07:23.8263497Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2880.json","@type":"CatalogPage","commitId":"b5a332b5-e2d1-457f-bab5-4b835300dde1","commitTimeStamp":"2017-10-17T05:38:35.9098609Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2881.json","@type":"CatalogPage","commitId":"475daf60-bfa4-4d2d-b97e-999eb7f7686a","commitTimeStamp":"2017-10-17T12:07:55.4459449Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2882.json","@type":"CatalogPage","commitId":"74ebbb83-7c8b-4281-b07f-bbd56dcd89b7","commitTimeStamp":"2017-10-17T17:26:13.5189869Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2883.json","@type":"CatalogPage","commitId":"d61ee9ec-4dab-4d3d-b356-28cafdca23cf","commitTimeStamp":"2017-10-17T22:53:45.3384631Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2884.json","@type":"CatalogPage","commitId":"deed69b1-8a5e-4de4-9aad-a92a5df7270f","commitTimeStamp":"2017-10-18T08:14:00.7659959Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2885.json","@type":"CatalogPage","commitId":"1d9aa8c5-b223-4cbe-bad6-924ca40a89c5","commitTimeStamp":"2017-10-18T12:18:48.2122326Z","count":536},{"@id":"https://api.nuget.org/v3/catalog0/page2886.json","@type":"CatalogPage","commitId":"3f9b47d4-0642-41c6-968f-ffb6ae07123f","commitTimeStamp":"2017-10-18T16:48:05.15643Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2887.json","@type":"CatalogPage","commitId":"c23b6c69-5468-4a9d-962e-22cd53cf18f6","commitTimeStamp":"2017-10-18T23:25:33.5578212Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page2888.json","@type":"CatalogPage","commitId":"341f16ae-aff4-4340-885b-4615af44bed3","commitTimeStamp":"2017-10-19T08:41:59.3433858Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2889.json","@type":"CatalogPage","commitId":"4c35684e-aa37-4a15-890a-fbfa5742b449","commitTimeStamp":"2017-10-19T15:30:38.8959092Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2890.json","@type":"CatalogPage","commitId":"6962b4b2-e5e4-4cad-9aac-7364ec0ebe03","commitTimeStamp":"2017-10-19T22:34:03.0603566Z","count":541},{"@id":"https://api.nuget.org/v3/catalog0/page2891.json","@type":"CatalogPage","commitId":"28add5fa-e830-4a09-b73f-f43534f5f23c","commitTimeStamp":"2017-10-20T07:59:20.7774948Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2892.json","@type":"CatalogPage","commitId":"0f599242-0330-4e9c-9af2-d9a06baec2cd","commitTimeStamp":"2017-10-20T14:12:36.4406004Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2893.json","@type":"CatalogPage","commitId":"81f5d5d2-60e8-4566-a8ca-48df0b382d28","commitTimeStamp":"2017-10-20T21:37:12.9855985Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2894.json","@type":"CatalogPage","commitId":"acb98442-f126-4379-a078-2cfe60d5ed59","commitTimeStamp":"2017-10-21T11:31:59.2095716Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2895.json","@type":"CatalogPage","commitId":"02da78ec-ca1e-4583-afb7-82f539ff9ffe","commitTimeStamp":"2017-10-22T08:14:41.3831163Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2896.json","@type":"CatalogPage","commitId":"a0720b78-f1ff-41bc-a9e6-e5f1548cf13c","commitTimeStamp":"2017-10-23T00:35:54.3321076Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page2897.json","@type":"CatalogPage","commitId":"c432e1f5-fbc7-4cc6-b002-d7487aa8032d","commitTimeStamp":"2017-10-23T08:38:51.7095095Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2898.json","@type":"CatalogPage","commitId":"a1489ea1-a619-4881-95f7-7b12e470e1d4","commitTimeStamp":"2017-10-23T14:27:46.8399722Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2899.json","@type":"CatalogPage","commitId":"ab47ec6a-d83e-4a80-bbdb-af4454d2f99d","commitTimeStamp":"2017-10-23T20:23:13.6972185Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2900.json","@type":"CatalogPage","commitId":"4c86fbdc-8831-475c-8997-636eb7c28ff3","commitTimeStamp":"2017-10-24T01:27:57.1718777Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2901.json","@type":"CatalogPage","commitId":"d66d7ff3-3638-47a3-98f9-018dc8d50976","commitTimeStamp":"2017-10-24T07:26:15.7982225Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2902.json","@type":"CatalogPage","commitId":"31f998a0-736a-47a6-8807-04022df2d6a3","commitTimeStamp":"2017-10-24T14:07:23.9664852Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2903.json","@type":"CatalogPage","commitId":"a11396d1-e1c3-4f26-bb43-3bd7b5b4ccbe","commitTimeStamp":"2017-10-24T18:57:53.769667Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2904.json","@type":"CatalogPage","commitId":"8d47425f-742a-4704-9f62-4938f3269e9b","commitTimeStamp":"2017-10-25T03:59:20.0030645Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page2905.json","@type":"CatalogPage","commitId":"071087e0-50cf-49b7-b65b-b1009bf91eb8","commitTimeStamp":"2017-10-25T10:06:23.5654476Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2906.json","@type":"CatalogPage","commitId":"041a0f2b-20cb-4b09-95b4-dd90fe060a5d","commitTimeStamp":"2017-10-25T14:41:26.3711337Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2907.json","@type":"CatalogPage","commitId":"bcf58161-eec8-4213-907e-1f2b9a5ecc79","commitTimeStamp":"2017-10-25T20:58:36.7353216Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2908.json","@type":"CatalogPage","commitId":"c3a9e519-7c62-456f-9ce7-4c0d7dc8cc1d","commitTimeStamp":"2017-10-26T08:42:42.8585347Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2909.json","@type":"CatalogPage","commitId":"a0ae6994-e68f-4647-b019-2c1ee6098243","commitTimeStamp":"2017-10-26T12:59:25.7197605Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2910.json","@type":"CatalogPage","commitId":"46380553-d6c6-4032-a801-0bae46bd7f3a","commitTimeStamp":"2017-10-26T16:59:22.5347518Z","count":542},{"@id":"https://api.nuget.org/v3/catalog0/page2911.json","@type":"CatalogPage","commitId":"e51d143d-20d7-40f7-a9fb-1862fb88b766","commitTimeStamp":"2017-10-26T23:19:56.3376766Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2912.json","@type":"CatalogPage","commitId":"77f013af-099a-4759-a457-e3a3e1adcbf0","commitTimeStamp":"2017-10-27T07:01:25.3352543Z","count":543},{"@id":"https://api.nuget.org/v3/catalog0/page2913.json","@type":"CatalogPage","commitId":"2765437a-b4e9-403f-800c-8adceff1dc57","commitTimeStamp":"2017-10-27T15:27:16.8725509Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2914.json","@type":"CatalogPage","commitId":"6addab96-2f95-4193-be85-a42b4a70c90d","commitTimeStamp":"2017-10-27T22:16:41.2904957Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2915.json","@type":"CatalogPage","commitId":"8621e4d4-a088-41dd-ad3a-0909095a9661","commitTimeStamp":"2017-10-28T18:19:10.0435684Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2916.json","@type":"CatalogPage","commitId":"7d1fff22-ae11-40d0-bec7-97366ae0928a","commitTimeStamp":"2017-10-29T08:50:20.4625234Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2917.json","@type":"CatalogPage","commitId":"521f6c48-ca95-4b3a-acf3-802aa67a03cc","commitTimeStamp":"2017-10-30T01:23:35.4445284Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2918.json","@type":"CatalogPage","commitId":"a6697af4-0054-4246-bc06-014a01460bbf","commitTimeStamp":"2017-10-30T08:47:36.4873999Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2919.json","@type":"CatalogPage","commitId":"f00e9f0f-3177-4fd5-bdec-e03b2b69bf2b","commitTimeStamp":"2017-10-30T13:25:22.635908Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2920.json","@type":"CatalogPage","commitId":"1321525a-7bee-4ee8-87a1-d1a193a92856","commitTimeStamp":"2017-10-30T17:27:55.1832099Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2921.json","@type":"CatalogPage","commitId":"86932e54-2360-4c69-b8d2-7da6f40983e1","commitTimeStamp":"2017-10-30T23:31:39.6747952Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2922.json","@type":"CatalogPage","commitId":"cc635227-1227-49f8-8e4b-48594aa836ea","commitTimeStamp":"2017-10-31T09:13:33.2502916Z","count":546},{"@id":"https://api.nuget.org/v3/catalog0/page2923.json","@type":"CatalogPage","commitId":"70383469-ef7b-43c1-a043-dbc9ae1247aa","commitTimeStamp":"2017-10-31T12:45:14.4086205Z","count":547},{"@id":"https://api.nuget.org/v3/catalog0/page2924.json","@type":"CatalogPage","commitId":"fcc0e991-b2a2-4b96-b466-630a5c6bfd80","commitTimeStamp":"2017-10-31T16:11:05.4189798Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2925.json","@type":"CatalogPage","commitId":"3ad0301d-f5f3-40e4-a5e3-b93d7e1f73a8","commitTimeStamp":"2017-10-31T22:08:31.3799214Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2926.json","@type":"CatalogPage","commitId":"b94fe609-3598-4a2f-93d5-dae5c5ba3516","commitTimeStamp":"2017-11-01T06:34:17.3578611Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2927.json","@type":"CatalogPage","commitId":"73757577-8b18-4852-95f9-e10965d4fb57","commitTimeStamp":"2017-11-01T07:10:01.5979547Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2928.json","@type":"CatalogPage","commitId":"cd47de1b-be48-4887-8c2c-7e8faf1b3d5a","commitTimeStamp":"2017-11-01T12:35:49.8375871Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2929.json","@type":"CatalogPage","commitId":"7e876a46-64bb-49df-b135-930d15eb6919","commitTimeStamp":"2017-11-01T17:03:44.2197429Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2930.json","@type":"CatalogPage","commitId":"c949a438-516b-4c82-b6ee-71979f5cb16b","commitTimeStamp":"2017-11-01T23:41:58.511788Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2931.json","@type":"CatalogPage","commitId":"d941bab5-0d0f-4670-ab77-5fd691be5c50","commitTimeStamp":"2017-11-02T07:02:22.8306219Z","count":545},{"@id":"https://api.nuget.org/v3/catalog0/page2932.json","@type":"CatalogPage","commitId":"0eeedad6-ca60-4e12-944f-9af18f15fe14","commitTimeStamp":"2017-11-02T11:21:13.183513Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2933.json","@type":"CatalogPage","commitId":"bd5a6b03-b691-4aa5-941a-4d430073cdd3","commitTimeStamp":"2017-11-02T16:13:10.1701543Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2934.json","@type":"CatalogPage","commitId":"9e658f43-d805-43b7-87fb-d710e20d6e31","commitTimeStamp":"2017-11-02T20:23:58.161543Z","count":544},{"@id":"https://api.nuget.org/v3/catalog0/page2935.json","@type":"CatalogPage","commitId":"c9963474-2d98-4f2d-a4ea-88bf0cc3d434","commitTimeStamp":"2017-11-03T04:20:32.4760307Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2936.json","@type":"CatalogPage","commitId":"112375cc-77c7-4ab7-ac34-6d3990d93d6c","commitTimeStamp":"2017-11-03T12:43:43.1367683Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2937.json","@type":"CatalogPage","commitId":"5c47519a-07d4-470d-8d14-9c9f1920ec66","commitTimeStamp":"2017-11-03T18:43:19.0789583Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2938.json","@type":"CatalogPage","commitId":"be0c75f8-05bc-456b-a597-d9a24bf710f3","commitTimeStamp":"2017-11-04T04:04:14.477004Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2939.json","@type":"CatalogPage","commitId":"883ca081-216c-4567-bce6-b8d663f62a85","commitTimeStamp":"2017-11-04T16:12:57.2557258Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2940.json","@type":"CatalogPage","commitId":"9b7e39ed-5342-4138-8e45-43d0690b75f8","commitTimeStamp":"2017-11-05T12:48:10.081492Z","count":550},{"@id":"https://api.nuget.org/v3/catalog0/page2941.json","@type":"CatalogPage","commitId":"084cff8d-7bd6-44dd-9646-6f996f97a4b7","commitTimeStamp":"2017-11-06T05:12:04.8612742Z","count":549},{"@id":"https://api.nuget.org/v3/catalog0/page2942.json","@type":"CatalogPage","commitId":"7ef6fc2e-7c3e-465a-a6f4-60aac3fe61ec","commitTimeStamp":"2017-11-06T13:07:34.4853812Z","count":548},{"@id":"https://api.nuget.org/v3/catalog0/page2943.json","@type":"CatalogPage","commitId":"ac845301-2ed7-4aa9-bba7-3c54c520419a","commitTimeStamp":"2017-11-06T18:10:27.926411Z","count":540},{"@id":"https://api.nuget.org/v3/catalog0/page2944.json","@type":"CatalogPage","commitId":"57de6c98-d4c6-4a24-95b9-1829c5013985","commitTimeStamp":"2017-11-06T19:30:56.0421411Z","count":112}],"@context":{"@vocab":"http://schema.nuget.org/catalog#","nuget":"http://schema.nuget.org/schema#","items":{"@id":"item","@container":"@set"},"parent":{"@type":"@id"},"commitTimeStamp":{"@type":"http://www.w3.org/2001/XMLSchema#dateTime"},"nuget:lastCreated":{"@type":"http://www.w3.org/2001/XMLSchema#dateTime"},"nuget:lastEdited":{"@type":"http://www.w3.org/2001/XMLSchema#dateTime"},"nuget:lastDeleted":{"@type":"http://www.w3.org/2001/XMLSchema#dateTime"}}} + + + https://api.nuget.org/v3/catalog0/index.json + + + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "3344hp", + "catalog:commitId": "eddb29f8-32c6-41da-8928-0940927a708b", + "catalog:commitTimeStamp": "2016-02-21T11:06:01.8896907Z", + "created": "2016-02-21T11:05:37.54Z", + "description": "Dingu.Generic.Repo.EF7 Class Library", + "frameworkAssemblyGroup": { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#frameworkassemblygroup/.netframework4.5.1", + "assembly": [ + "System.Runtime", + "mscorlib", + "System", + "System.Core", + "Microsoft.CSharp" + ], + "targetFramework": ".NETFramework4.5.1" + }, + "id": "Dingu.Generic.Repo.EF7", + "isPrerelease": true, + "lastEdited": "0001-01-01T00:00:00Z", + "licenseNames": "", + "licenseReportUrl": "", + "listed": true, + "packageHash": "xTgDmtAwG2VpwcZiD4bUxJin9S/Yutmixb7UfLWZMJR8l+LTAZsvoikwpDZgYkXlRnpAx/Wdgt/uaBLOpulFAg==", + "packageHashAlgorithm": "SHA512", + "packageSize": 7682, + "published": "2016-02-21T11:05:37.54Z", + "requireLicenseAcceptance": false, + "verbatimVersion": "1.0.0-beta2", + "version": "1.0.0-beta2", + "dependencyGroups": [ + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#dependencygroup/.netframework4.5.1", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#dependencygroup/.netframework4.5.1/entityframework.core", + "@type": "PackageDependency", + "id": "EntityFramework.Core", + "range": "[7.0.0-rc1-final, )" + }, + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#dependencygroup/.netframework4.5.1/entityframework.sqlite", + "@type": "PackageDependency", + "id": "EntityFramework.Sqlite", + "range": "[7.0.0-rc1-final, )" + }, + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#dependencygroup/.netframework4.5.1/nuget.commandline", + "@type": "PackageDependency", + "id": "NuGet.CommandLine", + "range": "[3.3.0, )" + }, + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#dependencygroup/.netframework4.5.1/system.runtime", + "@type": "PackageDependency", + "id": "System.Runtime", + "range": "[4.0.10, )" + } + ], + "targetFramework": ".NETFramework4.5.1" + }, + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#dependencygroup/.netplatform5.4", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#dependencygroup/.netplatform5.4/entityframework.core", + "@type": "PackageDependency", + "id": "EntityFramework.Core", + "range": "[7.0.0-rc1-final, )" + }, + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#dependencygroup/.netplatform5.4/entityframework.sqlite", + "@type": "PackageDependency", + "id": "EntityFramework.Sqlite", + "range": "[7.0.0-rc1-final, )" + }, + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#dependencygroup/.netplatform5.4/nuget.commandline", + "@type": "PackageDependency", + "id": "NuGet.CommandLine", + "range": "[3.3.0, )" + }, + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#dependencygroup/.netplatform5.4/system.runtime", + "@type": "PackageDependency", + "id": "System.Runtime", + "range": [ + "[4.0.10, )", + "[4.0.21-beta-23516, )" + ] + }, + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#dependencygroup/.netplatform5.4/microsoft.csharp", + "@type": "PackageDependency", + "id": "Microsoft.CSharp", + "range": "[4.0.1-beta-23516, )" + }, + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#dependencygroup/.netplatform5.4/system.collections", + "@type": "PackageDependency", + "id": "System.Collections", + "range": "[4.0.11-beta-23516, )" + }, + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#dependencygroup/.netplatform5.4/system.linq", + "@type": "PackageDependency", + "id": "System.Linq", + "range": "[4.0.1-beta-23516, )" + }, + { + "@id": "https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json#dependencygroup/.netplatform5.4/system.threading", + "@type": "PackageDependency", + "id": "System.Threading", + "range": "[4.0.11-beta-23516, )" + } + ], + "targetFramework": ".NETPlatform5.4" + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + } + } +} + + + https://api.nuget.org/v3/catalog0/data/2016.02.21.11.06.01/dingu.generic.repo.ef7.1.0.0-beta2.json + + + {"@id":"https://api.nuget.org/v3/registration3-gz-semver2/newtonsoft.json/12.0.1.json","@type":"nuget:PackageDetails","commitId":"47065e84-b83a-434f-9619-1b2f17df91b9","commitTimeStamp":"2019-10-10T00:00:00+00:00","nuget:id":"Newtonsoft.Json","nuget:version":"12.0.1"} + + + {"@id":"https://api.nuget.org/v3/catalog0/page2944.json","@type":"CatalogPage","commitId":"f241ce46-35ba-44c2-bd72-790eb44539a5","commitTimeStamp":"2017-11-06T22:07:49.3270578Z","count":218,"parent":"https://api.nuget.org/v3/catalog0/index.json","items":[{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.container.1.3.2.json","@type":"nuget:PackageDetails","commitId":"f241ce46-35ba-44c2-bd72-790eb44539a5","commitTimeStamp":"2017-11-06T22:07:49.3270578Z","nuget:id":"Dotnettency.Container","nuget:version":"1.3.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.middlewarepipeline.1.3.2.json","@type":"nuget:PackageDetails","commitId":"f241ce46-35ba-44c2-bd72-790eb44539a5","commitTimeStamp":"2017-11-06T22:07:49.3270578Z","nuget:id":"Dotnettency.MiddlewarePipeline","nuget:version":"1.3.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.container.structuremap.1.3.2.json","@type":"nuget:PackageDetails","commitId":"f241ce46-35ba-44c2-bd72-790eb44539a5","commitTimeStamp":"2017-11-06T22:07:49.3270578Z","nuget:id":"Dotnettency.Container.StructureMap","nuget:version":"1.3.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/momentum.pm.portalapi.5.10.16-legacy.json","@type":"nuget:PackageDetails","commitId":"f241ce46-35ba-44c2-bd72-790eb44539a5","commitTimeStamp":"2017-11-06T22:07:49.3270578Z","nuget:id":"Momentum.Pm.PortalApi","nuget:version":"5.10.16-legacy"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.hostingenvironment.1.3.2.json","@type":"nuget:PackageDetails","commitId":"f241ce46-35ba-44c2-bd72-790eb44539a5","commitTimeStamp":"2017-11-06T22:07:49.3270578Z","nuget:id":"Dotnettency.HostingEnvironment","nuget:version":"1.3.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/momentum.pm.api.5.10.16-legacy.json","@type":"nuget:PackageDetails","commitId":"f241ce46-35ba-44c2-bd72-790eb44539a5","commitTimeStamp":"2017-11-06T22:07:49.3270578Z","nuget:id":"Momentum.Pm.Api","nuget:version":"5.10.16-legacy"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.1.3.2.json","@type":"nuget:PackageDetails","commitId":"f241ce46-35ba-44c2-bd72-790eb44539a5","commitTimeStamp":"2017-11-06T22:07:49.3270578Z","nuget:id":"Dotnettency","nuget:version":"1.3.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.05.31/dappermagician.1.1.3.json","@type":"nuget:PackageDetails","commitId":"e2d9d60f-14dd-435e-baf4-7050761fa1e6","commitTimeStamp":"2017-11-06T22:05:31.0971396Z","nuget:id":"DapperMagician","nuget:version":"1.1.3"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.05.21/dotnettency.modules.nancy.1.4.0-unstable0039.json","@type":"nuget:PackageDetails","commitId":"cfa15e2b-30c9-471d-bb05-83f94ae8bf61","commitTimeStamp":"2017-11-06T22:05:21.596839Z","nuget:id":"Dotnettency.Modules.Nancy","nuget:version":"1.4.0-unstable0039"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.05.11/dotnettency.hostingenvironment.1.4.0-unstable0039.json","@type":"nuget:PackageDetails","commitId":"e19d3711-20a5-478f-8694-f556f3fcbf50","commitTimeStamp":"2017-11-06T22:05:11.8934453Z","nuget:id":"Dotnettency.HostingEnvironment","nuget:version":"1.4.0-unstable0039"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.05.11/dotnettency.container.1.4.0-unstable0039.json","@type":"nuget:PackageDetails","commitId":"e19d3711-20a5-478f-8694-f556f3fcbf50","commitTimeStamp":"2017-11-06T22:05:11.8934453Z","nuget:id":"Dotnettency.Container","nuget:version":"1.4.0-unstable0039"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.05.11/inetlab.smpp.1.2.8.json","@type":"nuget:PackageDetails","commitId":"e19d3711-20a5-478f-8694-f556f3fcbf50","commitTimeStamp":"2017-11-06T22:05:11.8934453Z","nuget:id":"Inetlab.SMPP","nuget:version":"1.2.8"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.05.11/momentum.pm.api.5.12.200-beta.json","@type":"nuget:PackageDetails","commitId":"e19d3711-20a5-478f-8694-f556f3fcbf50","commitTimeStamp":"2017-11-06T22:05:11.8934453Z","nuget:id":"Momentum.Pm.Api","nuget:version":"5.12.200-beta"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.05.11/vm.tools.1.1.0.json","@type":"nuget:PackageDetails","commitId":"e19d3711-20a5-478f-8694-f556f3fcbf50","commitTimeStamp":"2017-11-06T22:05:11.8934453Z","nuget:id":"Vm.Tools","nuget:version":"1.1.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.05.11/dotnettency.modules.1.4.0-unstable0039.json","@type":"nuget:PackageDetails","commitId":"e19d3711-20a5-478f-8694-f556f3fcbf50","commitTimeStamp":"2017-11-06T22:05:11.8934453Z","nuget:id":"Dotnettency.Modules","nuget:version":"1.4.0-unstable0039"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.05.11/momentum.pm.portalapi.5.12.200-beta.json","@type":"nuget:PackageDetails","commitId":"e19d3711-20a5-478f-8694-f556f3fcbf50","commitTimeStamp":"2017-11-06T22:05:11.8934453Z","nuget:id":"Momentum.Pm.PortalApi","nuget:version":"5.12.200-beta"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.05.11/dotnettency.middlewarepipeline.1.4.0-unstable0039.json","@type":"nuget:PackageDetails","commitId":"e19d3711-20a5-478f-8694-f556f3fcbf50","commitTimeStamp":"2017-11-06T22:05:11.8934453Z","nuget:id":"Dotnettency.MiddlewarePipeline","nuget:version":"1.4.0-unstable0039"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.05.11/dotnettency.container.structuremap.1.4.0-unstable0039.json","@type":"nuget:PackageDetails","commitId":"e19d3711-20a5-478f-8694-f556f3fcbf50","commitTimeStamp":"2017-11-06T22:05:11.8934453Z","nuget:id":"Dotnettency.Container.StructureMap","nuget:version":"1.4.0-unstable0039"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.05.11/momentum.pm.api.5.11.108.json","@type":"nuget:PackageDetails","commitId":"e19d3711-20a5-478f-8694-f556f3fcbf50","commitTimeStamp":"2017-11-06T22:05:11.8934453Z","nuget:id":"Momentum.Pm.Api","nuget:version":"5.11.108"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.05.11/dotnettency.1.4.0-unstable0039.json","@type":"nuget:PackageDetails","commitId":"e19d3711-20a5-478f-8694-f556f3fcbf50","commitTimeStamp":"2017-11-06T22:05:11.8934453Z","nuget:id":"Dotnettency","nuget:version":"1.4.0-unstable0039"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.05.11/momentum.pm.portalapi.5.11.108.json","@type":"nuget:PackageDetails","commitId":"e19d3711-20a5-478f-8694-f556f3fcbf50","commitTimeStamp":"2017-11-06T22:05:11.8934453Z","nuget:id":"Momentum.Pm.PortalApi","nuget:version":"5.11.108"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.02.55/carbon.versioning.1.4.0.json","@type":"nuget:PackageDetails","commitId":"e85b23fb-793c-41e8-ad62-44fcf574e02f","commitTimeStamp":"2017-11-06T22:02:55.0609451Z","nuget:id":"Carbon.Versioning","nuget:version":"1.4.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.00.38/carbon.kms.0.36.1.json","@type":"nuget:PackageDetails","commitId":"85f7a36d-0e80-4c2f-a1b4-4592f29f19d5","commitTimeStamp":"2017-11-06T22:00:38.0605673Z","nuget:id":"Carbon.Kms","nuget:version":"0.36.1"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.00.38/dappermagician.1.1.3.json","@type":"nuget:PackageDetails","commitId":"85f7a36d-0e80-4c2f-a1b4-4592f29f19d5","commitTimeStamp":"2017-11-06T22:00:38.0605673Z","nuget:id":"DapperMagician","nuget:version":"1.1.3"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.00.38/carbon.ci.0.46.6.json","@type":"nuget:PackageDetails","commitId":"85f7a36d-0e80-4c2f-a1b4-4592f29f19d5","commitTimeStamp":"2017-11-06T22:00:38.0605673Z","nuget:id":"Carbon.CI","nuget:version":"0.46.6"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.00.38/carbon.kms.abstractions.0.36.1.json","@type":"nuget:PackageDetails","commitId":"85f7a36d-0e80-4c2f-a1b4-4592f29f19d5","commitTimeStamp":"2017-11-06T22:00:38.0605673Z","nuget:id":"Carbon.Kms.Abstractions","nuget:version":"0.36.1"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.22.00.38/petprojects.framework.consul.0.9.1711.22.json","@type":"nuget:PackageDetails","commitId":"85f7a36d-0e80-4c2f-a1b4-4592f29f19d5","commitTimeStamp":"2017-11-06T22:00:38.0605673Z","nuget:id":"PetProjects.Framework.Consul","nuget:version":"0.9.1711.22"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.58.02/momentum.pm.portalapi.5.11.107.json","@type":"nuget:PackageDetails","commitId":"33e81d1f-efee-459d-b87f-dc57d8da56b4","commitTimeStamp":"2017-11-06T21:58:02.9394229Z","nuget:id":"Momentum.Pm.PortalApi","nuget:version":"5.11.107"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.57.38/momentum.pm.api.5.11.107.json","@type":"nuget:PackageDetails","commitId":"c003b8b3-0a26-4da4-ba2f-3b9d9d3c38ba","commitTimeStamp":"2017-11-06T21:57:38.0792646Z","nuget:id":"Momentum.Pm.Api","nuget:version":"5.11.107"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.57.09/naos.recipes.runwithretry.1.0.78.json","@type":"nuget:PackageDetails","commitId":"af713ba0-3041-423a-aafb-c645a2575a7b","commitTimeStamp":"2017-11-06T21:57:09.2657909Z","nuget:id":"Naos.Recipes.RunWithRetry","nuget:version":"1.0.78"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.57.09/naos.recipes.console.exampleconfig.1.0.78.json","@type":"nuget:PackageDetails","commitId":"af713ba0-3041-423a-aafb-c645a2575a7b","commitTimeStamp":"2017-11-06T21:57:09.2657909Z","nuget:id":"Naos.Recipes.Console.ExampleConfig","nuget:version":"1.0.78"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.57.09/naos.recipes.console.domain.1.0.78.json","@type":"nuget:PackageDetails","commitId":"af713ba0-3041-423a-aafb-c645a2575a7b","commitTimeStamp":"2017-11-06T21:57:09.2657909Z","nuget:id":"Naos.Recipes.Console.Domain","nuget:version":"1.0.78"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.57.09/momentum.pm.portalapi.5.12.199-beta.json","@type":"nuget:PackageDetails","commitId":"af713ba0-3041-423a-aafb-c645a2575a7b","commitTimeStamp":"2017-11-06T21:57:09.2657909Z","nuget:id":"Momentum.Pm.PortalApi","nuget:version":"5.12.199-beta"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.57.09/naos.recipes.initializetestproject.1.0.78.json","@type":"nuget:PackageDetails","commitId":"af713ba0-3041-423a-aafb-c645a2575a7b","commitTimeStamp":"2017-11-06T21:57:09.2657909Z","nuget:id":"Naos.Recipes.InitializeTestProject","nuget:version":"1.0.78"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.57.09/naos.recipes.tupleinitializers.1.0.78.json","@type":"nuget:PackageDetails","commitId":"af713ba0-3041-423a-aafb-c645a2575a7b","commitTimeStamp":"2017-11-06T21:57:09.2657909Z","nuget:id":"Naos.Recipes.TupleInitializers","nuget:version":"1.0.78"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.57.09/momentum.pm.api.5.12.199-beta.json","@type":"nuget:PackageDetails","commitId":"af713ba0-3041-423a-aafb-c645a2575a7b","commitTimeStamp":"2017-11-06T21:57:09.2657909Z","nuget:id":"Momentum.Pm.Api","nuget:version":"5.12.199-beta"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.57.09/naos.recipes.deployment.commonconfig.1.0.78.json","@type":"nuget:PackageDetails","commitId":"af713ba0-3041-423a-aafb-c645a2575a7b","commitTimeStamp":"2017-11-06T21:57:09.2657909Z","nuget:id":"Naos.Recipes.Deployment.CommonConfig","nuget:version":"1.0.78"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.57.09/naos.recipes.cryptography.hashing.1.0.78.json","@type":"nuget:PackageDetails","commitId":"af713ba0-3041-423a-aafb-c645a2575a7b","commitTimeStamp":"2017-11-06T21:57:09.2657909Z","nuget:id":"Naos.Recipes.Cryptography.Hashing","nuget:version":"1.0.78"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.57.09/naos.recipes.itsdomain.authorization.1.0.78.json","@type":"nuget:PackageDetails","commitId":"af713ba0-3041-423a-aafb-c645a2575a7b","commitTimeStamp":"2017-11-06T21:57:09.2657909Z","nuget:id":"Naos.Recipes.ItsDomain.Authorization","nuget:version":"1.0.78"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.54.32/securestrconvertor.varun_rusiya.1.0.0.9.json","@type":"nuget:PackageDetails","commitId":"b9aec44a-e9a5-4e03-9d75-088af3f32ec1","commitTimeStamp":"2017-11-06T21:54:32.6667795Z","nuget:id":"SecureStrConvertor.VARUN_RUSIYA","nuget:version":"1.0.0.9"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.54.22/naos.recipes.console.bootstrapper.1.0.78.json","@type":"nuget:PackageDetails","commitId":"f376901c-415e-48bf-9cab-f6ea0b7bd82c","commitTimeStamp":"2017-11-06T21:54:22.9789114Z","nuget:id":"Naos.Recipes.Console.Bootstrapper","nuget:version":"1.0.78"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.54.13/naos.recipes.configuration.setup.1.0.78.json","@type":"nuget:PackageDetails","commitId":"d1b111aa-7e99-41bc-b5f3-18ef9225ccca","commitTimeStamp":"2017-11-06T21:54:13.38485Z","nuget:id":"Naos.Recipes.Configuration.Setup","nuget:version":"1.0.78"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.54.04/naos.recipes.configuration.setup.1.0.77.json","@type":"nuget:PackageDetails","commitId":"fecb9cac-cb09-47ce-8966-99881aa124b8","commitTimeStamp":"2017-11-06T21:54:04.7439456Z","nuget:id":"Naos.Recipes.Configuration.Setup","nuget:version":"1.0.77"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.54.04/naos.recipes.console.bootstrapper.1.0.77.json","@type":"nuget:PackageDetails","commitId":"fecb9cac-cb09-47ce-8966-99881aa124b8","commitTimeStamp":"2017-11-06T21:54:04.7439456Z","nuget:id":"Naos.Recipes.Console.Bootstrapper","nuget:version":"1.0.77"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.51.49/momentum.pm.api.5.10.15-legacy.json","@type":"nuget:PackageDetails","commitId":"43892d0b-c3aa-46ac-b1e6-11d3317aeaee","commitTimeStamp":"2017-11-06T21:51:49.2395561Z","nuget:id":"Momentum.Pm.Api","nuget:version":"5.10.15-legacy"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.51.49/sourcecode.chasm.repository.hybrid.1.0.0-preview1-00218.json","@type":"nuget:PackageDetails","commitId":"43892d0b-c3aa-46ac-b1e6-11d3317aeaee","commitTimeStamp":"2017-11-06T21:51:49.2395561Z","nuget:id":"SourceCode.Chasm.Repository.Hybrid","nuget:version":"1.0.0-preview1-00218"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.51.49/sourcecode.chasm.repository.disk.1.0.0-preview1-00218.json","@type":"nuget:PackageDetails","commitId":"43892d0b-c3aa-46ac-b1e6-11d3317aeaee","commitTimeStamp":"2017-11-06T21:51:49.2395561Z","nuget:id":"SourceCode.Chasm.Repository.Disk","nuget:version":"1.0.0-preview1-00218"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.51.49/developmenthelpers.sqllogging.2.0.4.json","@type":"nuget:PackageDetails","commitId":"43892d0b-c3aa-46ac-b1e6-11d3317aeaee","commitTimeStamp":"2017-11-06T21:51:49.2395561Z","nuget:id":"DevelopmentHelpers.SqlLogging","nuget:version":"2.0.4"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.51.49/sourcecode.chasm.io.json.1.0.0-preview1-00218.json","@type":"nuget:PackageDetails","commitId":"43892d0b-c3aa-46ac-b1e6-11d3317aeaee","commitTimeStamp":"2017-11-06T21:51:49.2395561Z","nuget:id":"SourceCode.Chasm.IO.Json","nuget:version":"1.0.0-preview1-00218"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.51.49/sourcecode.chasm.repository.azureblob.1.0.0-preview1-00218.json","@type":"nuget:PackageDetails","commitId":"43892d0b-c3aa-46ac-b1e6-11d3317aeaee","commitTimeStamp":"2017-11-06T21:51:49.2395561Z","nuget:id":"SourceCode.Chasm.Repository.AzureBlob","nuget:version":"1.0.0-preview1-00218"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.51.49/sourcecode.chasm.repository.azuretable.1.0.0-preview1-00218.json","@type":"nuget:PackageDetails","commitId":"43892d0b-c3aa-46ac-b1e6-11d3317aeaee","commitTimeStamp":"2017-11-06T21:51:49.2395561Z","nuget:id":"SourceCode.Chasm.Repository.AzureTable","nuget:version":"1.0.0-preview1-00218"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.51.49/sourcecode.chasm.io.proto.1.0.0-preview1-00218.json","@type":"nuget:PackageDetails","commitId":"43892d0b-c3aa-46ac-b1e6-11d3317aeaee","commitTimeStamp":"2017-11-06T21:51:49.2395561Z","nuget:id":"SourceCode.Chasm.IO.Proto","nuget:version":"1.0.0-preview1-00218"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.51.49/sourcecode.chasm.1.0.0-preview1-00218.json","@type":"nuget:PackageDetails","commitId":"43892d0b-c3aa-46ac-b1e6-11d3317aeaee","commitTimeStamp":"2017-11-06T21:51:49.2395561Z","nuget:id":"SourceCode.Chasm","nuget:version":"1.0.0-preview1-00218"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.51.49/sourcecode.chasm.io.text.1.0.0-preview1-00218.json","@type":"nuget:PackageDetails","commitId":"43892d0b-c3aa-46ac-b1e6-11d3317aeaee","commitTimeStamp":"2017-11-06T21:51:49.2395561Z","nuget:id":"SourceCode.Chasm.IO.Text","nuget:version":"1.0.0-preview1-00218"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.51.49/momentum.pm.portalapi.5.10.15-legacy.json","@type":"nuget:PackageDetails","commitId":"43892d0b-c3aa-46ac-b1e6-11d3317aeaee","commitTimeStamp":"2017-11-06T21:51:49.2395561Z","nuget:id":"Momentum.Pm.PortalApi","nuget:version":"5.10.15-legacy"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.51.49/securestrconvertor.varun_rusiya.1.0.0.9.json","@type":"nuget:PackageDetails","commitId":"43892d0b-c3aa-46ac-b1e6-11d3317aeaee","commitTimeStamp":"2017-11-06T21:51:49.2395561Z","nuget:id":"SecureStrConvertor.VARUN_RUSIYA","nuget:version":"1.0.0.9"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.49.29/developmenthelpers.sqllogging.2.0.3.json","@type":"nuget:PackageDetails","commitId":"41aa9e9b-fea2-4578-abb7-c772025c63b0","commitTimeStamp":"2017-11-06T21:49:29.7872051Z","nuget:id":"DevelopmentHelpers.SqlLogging","nuget:version":"2.0.3"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.49.07/momentum.pm.portalapi.5.12.198-beta.json","@type":"nuget:PackageDetails","commitId":"4cd8fc4c-869d-4f21-8bc1-f7ec30bb3e66","commitTimeStamp":"2017-11-06T21:49:07.6146624Z","nuget:id":"Momentum.Pm.PortalApi","nuget:version":"5.12.198-beta"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.48.43/momentum.pm.api.5.12.198-beta.json","@type":"nuget:PackageDetails","commitId":"b8e04137-aff3-48ab-a2b1-48f55a51c766","commitTimeStamp":"2017-11-06T21:48:43.3950747Z","nuget:id":"Momentum.Pm.Api","nuget:version":"5.12.198-beta"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.48.43/dotnettency.modules.nancy.1.4.0-unstable0035.json","@type":"nuget:PackageDetails","commitId":"b8e04137-aff3-48ab-a2b1-48f55a51c766","commitTimeStamp":"2017-11-06T21:48:43.3950747Z","nuget:id":"Dotnettency.Modules.Nancy","nuget:version":"1.4.0-unstable0035"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.48.43/dotnettency.modules.1.4.0-unstable0035.json","@type":"nuget:PackageDetails","commitId":"b8e04137-aff3-48ab-a2b1-48f55a51c766","commitTimeStamp":"2017-11-06T21:48:43.3950747Z","nuget:id":"Dotnettency.Modules","nuget:version":"1.4.0-unstable0035"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.48.14/developmenthelpers.sqllogging.2.0.3.json","@type":"nuget:PackageDetails","commitId":"39759890-5824-4dee-a3e5-ca63a15398ec","commitTimeStamp":"2017-11-06T21:48:14.9253644Z","nuget:id":"DevelopmentHelpers.SqlLogging","nuget:version":"2.0.3"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.48.14/dotnettency.middlewarepipeline.1.4.0-unstable0035.json","@type":"nuget:PackageDetails","commitId":"39759890-5824-4dee-a3e5-ca63a15398ec","commitTimeStamp":"2017-11-06T21:48:14.9253644Z","nuget:id":"Dotnettency.MiddlewarePipeline","nuget:version":"1.4.0-unstable0035"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.48.14/dotnettency.hostingenvironment.1.4.0-unstable0035.json","@type":"nuget:PackageDetails","commitId":"39759890-5824-4dee-a3e5-ca63a15398ec","commitTimeStamp":"2017-11-06T21:48:14.9253644Z","nuget:id":"Dotnettency.HostingEnvironment","nuget:version":"1.4.0-unstable0035"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.47.46/gdaxapi.0.0.27.json","@type":"nuget:PackageDetails","commitId":"081de23f-abab-450b-b63b-562e63d4e7b1","commitTimeStamp":"2017-11-06T21:47:46.8932519Z","nuget:id":"GdaxApi","nuget:version":"0.0.27"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.47.46/dotnettency.container.1.4.0-unstable0035.json","@type":"nuget:PackageDetails","commitId":"081de23f-abab-450b-b63b-562e63d4e7b1","commitTimeStamp":"2017-11-06T21:47:46.8932519Z","nuget:id":"Dotnettency.Container","nuget:version":"1.4.0-unstable0035"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.47.46/dotnettency.container.structuremap.1.4.0-unstable0035.json","@type":"nuget:PackageDetails","commitId":"081de23f-abab-450b-b63b-562e63d4e7b1","commitTimeStamp":"2017-11-06T21:47:46.8932519Z","nuget:id":"Dotnettency.Container.StructureMap","nuget:version":"1.4.0-unstable0035"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.47.46/dotnettency.1.4.0-unstable0035.json","@type":"nuget:PackageDetails","commitId":"081de23f-abab-450b-b63b-562e63d4e7b1","commitTimeStamp":"2017-11-06T21:47:46.8932519Z","nuget:id":"Dotnettency","nuget:version":"1.4.0-unstable0035"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.47.46/momentum.pm.portalapi.5.10.14-legacy.json","@type":"nuget:PackageDetails","commitId":"081de23f-abab-450b-b63b-562e63d4e7b1","commitTimeStamp":"2017-11-06T21:47:46.8932519Z","nuget:id":"Momentum.Pm.PortalApi","nuget:version":"5.10.14-legacy"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.47.46/momentum.pm.api.5.10.14-legacy.json","@type":"nuget:PackageDetails","commitId":"081de23f-abab-450b-b63b-562e63d4e7b1","commitTimeStamp":"2017-11-06T21:47:46.8932519Z","nuget:id":"Momentum.Pm.Api","nuget:version":"5.10.14-legacy"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.45.11/microsoft.reportingservices.reportviewercontrol.winforms.140.1000.523.json","@type":"nuget:PackageDetails","commitId":"b889a6e0-e1c9-4a8a-88e1-621fd49c41e2","commitTimeStamp":"2017-11-06T21:45:11.1210826Z","nuget:id":"Microsoft.ReportingServices.ReportViewerControl.Winforms","nuget:version":"140.1000.523"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.42.52/microsoft.reportingservices.reportviewercontrol.webforms.140.1000.523.json","@type":"nuget:PackageDetails","commitId":"e078407c-3dfd-4dae-a8c2-c136d01817f9","commitTimeStamp":"2017-11-06T21:42:52.4917023Z","nuget:id":"Microsoft.ReportingServices.ReportViewerControl.WebForms","nuget:version":"140.1000.523"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.40.36/mediadevices.1.4.0.json","@type":"nuget:PackageDetails","commitId":"c6889508-1296-47a1-a0f2-da748d3d65c1","commitTimeStamp":"2017-11-06T21:40:36.3769676Z","nuget:id":"MediaDevices","nuget:version":"1.4.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.40.36/wpfclipboardmonitor.1.0.0.json","@type":"nuget:PackageDetails","commitId":"c6889508-1296-47a1-a0f2-da748d3d65c1","commitTimeStamp":"2017-11-06T21:40:36.3769676Z","nuget:id":"WpfClipboardMonitor","nuget:version":"1.0.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.40.26/momentum.pm.portalapi.5.11.106.json","@type":"nuget:PackageDetails","commitId":"c1dbf254-82c3-4af8-9f4d-f8179d1a2aa5","commitTimeStamp":"2017-11-06T21:40:26.7516582Z","nuget:id":"Momentum.Pm.PortalApi","nuget:version":"5.11.106"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.40.26/momentum.pm.api.5.11.106.json","@type":"nuget:PackageDetails","commitId":"c1dbf254-82c3-4af8-9f4d-f8179d1a2aa5","commitTimeStamp":"2017-11-06T21:40:26.7516582Z","nuget:id":"Momentum.Pm.Api","nuget:version":"5.11.106"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.38.09/momentum.pm.api.5.11.105.json","@type":"nuget:PackageDetails","commitId":"62d83144-6384-484f-afe0-09e6e74b5fd9","commitTimeStamp":"2017-11-06T21:38:09.2003495Z","nuget:id":"Momentum.Pm.Api","nuget:version":"5.11.105"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.38.09/momentum.pm.portalapi.5.11.105.json","@type":"nuget:PackageDetails","commitId":"62d83144-6384-484f-afe0-09e6e74b5fd9","commitTimeStamp":"2017-11-06T21:38:09.2003495Z","nuget:id":"Momentum.Pm.PortalApi","nuget:version":"5.11.105"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.35.52/inflatable.1.0.50.json","@type":"nuget:PackageDetails","commitId":"941103bd-e749-4403-86f5-7519706ad9a2","commitTimeStamp":"2017-11-06T21:35:52.3363903Z","nuget:id":"Inflatable","nuget:version":"1.0.50"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.35.52/unity.abstractions.2.1.1.json","@type":"nuget:PackageDetails","commitId":"941103bd-e749-4403-86f5-7519706ad9a2","commitTimeStamp":"2017-11-06T21:35:52.3363903Z","nuget:id":"Unity.Abstractions","nuget:version":"2.1.1"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.35.52/zenseless.0.3.6.json","@type":"nuget:PackageDetails","commitId":"941103bd-e749-4403-86f5-7519706ad9a2","commitTimeStamp":"2017-11-06T21:35:52.3363903Z","nuget:id":"Zenseless","nuget:version":"0.3.6"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.33.32/momentum.pm.api.5.12.197-beta.json","@type":"nuget:PackageDetails","commitId":"c99efd36-6f7e-4264-a810-52196258d958","commitTimeStamp":"2017-11-06T21:33:32.3574864Z","nuget:id":"Momentum.Pm.Api","nuget:version":"5.12.197-beta"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.33.32/sourcecode.clay.threading.1.0.0-preview1-00278.json","@type":"nuget:PackageDetails","commitId":"c99efd36-6f7e-4264-a810-52196258d958","commitTimeStamp":"2017-11-06T21:33:32.3574864Z","nuget:id":"SourceCode.Clay.Threading","nuget:version":"1.0.0-preview1-00278"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.33.32/momentum.pm.portalapi.5.12.197-beta.json","@type":"nuget:PackageDetails","commitId":"c99efd36-6f7e-4264-a810-52196258d958","commitTimeStamp":"2017-11-06T21:33:32.3574864Z","nuget:id":"Momentum.Pm.PortalApi","nuget:version":"5.12.197-beta"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.31.13/homewizard.net.1.0.3.json","@type":"nuget:PackageDetails","commitId":"9dfdd7f8-5295-447f-940f-f59977fdd6cd","commitTimeStamp":"2017-11-06T21:31:13.622101Z","nuget:id":"HomeWizard.Net","nuget:version":"1.0.3"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.31.13/write-progressex.0.17.0.json","@type":"nuget:PackageDetails","commitId":"9dfdd7f8-5295-447f-940f-f59977fdd6cd","commitTimeStamp":"2017-11-06T21:31:13.622101Z","nuget:id":"Write-ProgressEx","nuget:version":"0.17.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.31.04/sourcecode.clay.text.1.0.0-preview1-00278.json","@type":"nuget:PackageDetails","commitId":"c589c2d3-7652-4cd6-bf80-46b03344e956","commitTimeStamp":"2017-11-06T21:31:04.1055481Z","nuget:id":"SourceCode.Clay.Text","nuget:version":"1.0.0-preview1-00278"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.30.55/sourcecode.clay.openapi.1.0.0-preview1-00278.json","@type":"nuget:PackageDetails","commitId":"4c840181-9967-4825-b770-ef2773d1f713","commitTimeStamp":"2017-11-06T21:30:55.0195475Z","nuget:id":"SourceCode.Clay.OpenApi","nuget:version":"1.0.0-preview1-00278"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.30.45/sourcecode.clay.json.1.0.0-preview1-00278.json","@type":"nuget:PackageDetails","commitId":"90b3b9e3-68db-48bf-9d45-0cc70ce9bed4","commitTimeStamp":"2017-11-06T21:30:45.7379688Z","nuget:id":"SourceCode.Clay.Json","nuget:version":"1.0.0-preview1-00278"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.30.36/sourcecode.clay.data.1.0.0-preview1-00278.json","@type":"nuget:PackageDetails","commitId":"7078dd2b-7831-4483-b614-bb1f4327e1a3","commitTimeStamp":"2017-11-06T21:30:36.9564653Z","nuget:id":"SourceCode.Clay.Data","nuget:version":"1.0.0-preview1-00278"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.30.36/storm.buildtasks.androidcolors.0.2.2.json","@type":"nuget:PackageDetails","commitId":"7078dd2b-7831-4483-b614-bb1f4327e1a3","commitTimeStamp":"2017-11-06T21:30:36.9564653Z","nuget:id":"Storm.BuildTasks.AndroidColors","nuget:version":"0.2.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.30.28/sourcecode.clay.collections.1.0.0-preview1-00278.json","@type":"nuget:PackageDetails","commitId":"87f51383-353b-4ec4-bbf2-b9aeab9ffa01","commitTimeStamp":"2017-11-06T21:30:28.0452768Z","nuget:id":"SourceCode.Clay.Collections","nuget:version":"1.0.0-preview1-00278"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.30.17/sourcecode.clay.1.0.0-preview1-00278.json","@type":"nuget:PackageDetails","commitId":"1dcf6f35-bbd0-4c96-8d30-51550a8ae21e","commitTimeStamp":"2017-11-06T21:30:17.9320566Z","nuget:id":"SourceCode.Clay","nuget:version":"1.0.0-preview1-00278"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.30.17/makesensframeworkcoreservices.2017.11.6.3.json","@type":"nuget:PackageDetails","commitId":"1dcf6f35-bbd0-4c96-8d30-51550a8ae21e","commitTimeStamp":"2017-11-06T21:30:17.9320566Z","nuget:id":"MakeSensFrameworkCoreServices","nuget:version":"2017.11.6.3"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.30.17/sourcecode.clay.buffers.1.0.0-preview1-00278.json","@type":"nuget:PackageDetails","commitId":"1dcf6f35-bbd0-4c96-8d30-51550a8ae21e","commitTimeStamp":"2017-11-06T21:30:17.9320566Z","nuget:id":"SourceCode.Clay.Buffers","nuget:version":"1.0.0-preview1-00278"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.27.57/med.configuracion.api.1.3.0.json","@type":"nuget:PackageDetails","commitId":"bd823677-1314-4e0a-ac48-85b4a586f12c","commitTimeStamp":"2017-11-06T21:27:57.955022Z","nuget:id":"Med.Configuracion.API","nuget:version":"1.3.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.27.57/homewizard.net.1.0.3.json","@type":"nuget:PackageDetails","commitId":"bd823677-1314-4e0a-ac48-85b4a586f12c","commitTimeStamp":"2017-11-06T21:27:57.955022Z","nuget:id":"HomeWizard.Net","nuget:version":"1.0.3"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.27.57/serilog.sinks.awscloudwatch.3.0.86.json","@type":"nuget:PackageDetails","commitId":"bd823677-1314-4e0a-ac48-85b4a586f12c","commitTimeStamp":"2017-11-06T21:27:57.955022Z","nuget:id":"Serilog.Sinks.AwsCloudWatch","nuget:version":"3.0.86"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.27.57/write-progressex.0.17.0.json","@type":"nuget:PackageDetails","commitId":"bd823677-1314-4e0a-ac48-85b4a586f12c","commitTimeStamp":"2017-11-06T21:27:57.955022Z","nuget:id":"Write-ProgressEx","nuget:version":"0.17.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.25.38/fluentregistration.2.5.1.json","@type":"nuget:PackageDetails","commitId":"e3d0105c-8fb6-4d61-af52-b79e56943532","commitTimeStamp":"2017-11-06T21:25:38.2091749Z","nuget:id":"FluentRegistration","nuget:version":"2.5.1"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.25.38/azurite.1.1.0.json","@type":"nuget:PackageDetails","commitId":"e3d0105c-8fb6-4d61-af52-b79e56943532","commitTimeStamp":"2017-11-06T21:25:38.2091749Z","nuget:id":"Azurite","nuget:version":"1.1.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.23.17/lykke.service.clientaccount.client.1.0.43-beta.json","@type":"nuget:PackageDetails","commitId":"38010446-604a-4a17-be0c-699513eed50c","commitTimeStamp":"2017-11-06T21:23:17.0080684Z","nuget:id":"Lykke.Service.ClientAccount.Client","nuget:version":"1.0.43-beta"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.18.41/securestrconvertor.varun_rusiya.1.0.0.8.json","@type":"nuget:PackageDetails","commitId":"0f5d4617-2122-43be-8d71-056d8dae489c","commitTimeStamp":"2017-11-06T21:18:41.1226055Z","nuget:id":"SecureStrConvertor.VARUN_RUSIYA","nuget:version":"1.0.0.8"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.18.31/rubberstamp.0.1.0.4.json","@type":"nuget:PackageDetails","commitId":"3d83b571-2a51-4eff-a3fa-05671c1aff4f","commitTimeStamp":"2017-11-06T21:18:31.9504356Z","nuget:id":"RubberStamp","nuget:version":"0.1.0.4"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.18.31/manatee.json.9.0.0.json","@type":"nuget:PackageDetails","commitId":"3d83b571-2a51-4eff-a3fa-05671c1aff4f","commitTimeStamp":"2017-11-06T21:18:31.9504356Z","nuget:id":"Manatee.Json","nuget:version":"9.0.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.21.18.31/securestrconvertor.varun_rusiya.1.0.0.8.json","@type":"nuget:PackageDetails","commitId":"3d83b571-2a51-4eff-a3fa-05671c1aff4f","commitTimeStamp":"2017-11-06T21:18:31.9504356Z","nuget:id":"SecureStrConvertor.VARUN_RUSIYA","nuget:version":"1.0.0.8"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.30.56/meziantou.framework.1.0.16.json","@type":"nuget:PackageDetails","commitId":"57de6c98-d4c6-4a24-95b9-1829c5013985","commitTimeStamp":"2017-11-06T19:30:56.0421411Z","nuget:id":"Meziantou.Framework","nuget:version":"1.0.16"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.30.56/makesensframeworkcoreservices.2017.11.6.2.json","@type":"nuget:PackageDetails","commitId":"57de6c98-d4c6-4a24-95b9-1829c5013985","commitTimeStamp":"2017-11-06T19:30:56.0421411Z","nuget:id":"MakeSensFrameworkCoreServices","nuget:version":"2017.11.6.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.30.56/crossroads.web.common.1.1.10.json","@type":"nuget:PackageDetails","commitId":"57de6c98-d4c6-4a24-95b9-1829c5013985","commitTimeStamp":"2017-11-06T19:30:56.0421411Z","nuget:id":"Crossroads.Web.Common","nuget:version":"1.1.10"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.30.56/genexcel.0.2.0.json","@type":"nuget:PackageDetails","commitId":"57de6c98-d4c6-4a24-95b9-1829c5013985","commitTimeStamp":"2017-11-06T19:30:56.0421411Z","nuget:id":"Genexcel","nuget:version":"0.2.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.30.56/meziantou.framework.codedom.1.0.8.json","@type":"nuget:PackageDetails","commitId":"57de6c98-d4c6-4a24-95b9-1829c5013985","commitTimeStamp":"2017-11-06T19:30:56.0421411Z","nuget:id":"Meziantou.Framework.CodeDom","nuget:version":"1.0.8"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.29.03/microsoft.azure.iot.edge.function.0.3.0.json","@type":"nuget:PackageDelete","commitId":"b429c4e3-5127-4430-9cc8-74927c9c3886","commitTimeStamp":"2017-11-06T19:29:03.6198426Z","nuget:id":"Microsoft.Azure.IoT.Edge.Function","nuget:version":"0.3.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.29.03/microsoft.azure.iot.edge.function.0.4.0.json","@type":"nuget:PackageDelete","commitId":"b429c4e3-5127-4430-9cc8-74927c9c3886","commitTimeStamp":"2017-11-06T19:29:03.6198426Z","nuget:id":"Microsoft.Azure.IoT.Edge.Function","nuget:version":"0.4.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.29.03/microsoft.azure.iot.edge.function.0.5.0.json","@type":"nuget:PackageDelete","commitId":"b429c4e3-5127-4430-9cc8-74927c9c3886","commitTimeStamp":"2017-11-06T19:29:03.6198426Z","nuget:id":"Microsoft.Azure.IoT.Edge.Function","nuget:version":"0.5.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.29.03/microsoft.azure.iot.edge.function.0.2.0.json","@type":"nuget:PackageDelete","commitId":"b429c4e3-5127-4430-9cc8-74927c9c3886","commitTimeStamp":"2017-11-06T19:29:03.6198426Z","nuget:id":"Microsoft.Azure.IoT.Edge.Function","nuget:version":"0.2.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.29.03/microsoft.azure.iot.edge.function.0.6.0.json","@type":"nuget:PackageDelete","commitId":"b429c4e3-5127-4430-9cc8-74927c9c3886","commitTimeStamp":"2017-11-06T19:29:03.6198426Z","nuget:id":"Microsoft.Azure.IoT.Edge.Function","nuget:version":"0.6.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.22.12/iccorp.infraestructura.repositorio.rabbitmq.1.0.0.7.json","@type":"nuget:PackageDetails","commitId":"68db1da1-68f9-47f3-9a74-23254d361ea8","commitTimeStamp":"2017-11-06T19:22:12.996134Z","nuget:id":"ICCorp.Infraestructura.Repositorio.RabbitMQ","nuget:version":"1.0.0.7"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.17.32/wkx.0.4.0.json","@type":"nuget:PackageDetails","commitId":"7ead9009-ba51-4708-84ab-fd79b77570a4","commitTimeStamp":"2017-11-06T19:17:32.6743228Z","nuget:id":"Wkx","nuget:version":"0.4.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.15.07/lykke.rabbitmqbroker.4.0.4.json","@type":"nuget:PackageDetails","commitId":"bd4622e0-268d-4091-9d0f-b58f5ceb715d","commitTimeStamp":"2017-11-06T19:15:07.1017103Z","nuget:id":"Lykke.RabbitMqBroker","nuget:version":"4.0.4"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.15.07/melanchall.drywetmidi.2.0.0.json","@type":"nuget:PackageDetails","commitId":"bd4622e0-268d-4091-9d0f-b58f5ceb715d","commitTimeStamp":"2017-11-06T19:15:07.1017103Z","nuget:id":"Melanchall.DryWetMidi","nuget:version":"2.0.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.12.40/lifeimage-agent.1.0.19.json","@type":"nuget:PackageDetails","commitId":"704a37e7-ac32-4fcf-86b4-167f020c76c7","commitTimeStamp":"2017-11-06T19:12:40.5339344Z","nuget:id":"lifeimage-agent","nuget:version":"1.0.19"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.10.16/lykke.rabbitmqbroker.4.0.3.json","@type":"nuget:PackageDetails","commitId":"4577e271-348a-4f3f-ae30-5d566dce6e35","commitTimeStamp":"2017-11-06T19:10:16.0983741Z","nuget:id":"Lykke.RabbitMqBroker","nuget:version":"4.0.3"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.05.31/naos.packaging.nuget.1.0.36.json","@type":"nuget:PackageDetails","commitId":"b8853431-9b69-487d-ab7a-d7c4dd8e59d0","commitTimeStamp":"2017-11-06T19:05:31.6034108Z","nuget:id":"Naos.Packaging.NuGet","nuget:version":"1.0.36"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.05.31/pickles.commandline.2.17.0.json","@type":"nuget:PackageDetails","commitId":"b8853431-9b69-487d-ab7a-d7c4dd8e59d0","commitTimeStamp":"2017-11-06T19:05:31.6034108Z","nuget:id":"Pickles.CommandLine","nuget:version":"2.17.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.05.31/naos.packaging.domain.1.0.36.json","@type":"nuget:PackageDetails","commitId":"b8853431-9b69-487d-ab7a-d7c4dd8e59d0","commitTimeStamp":"2017-11-06T19:05:31.6034108Z","nuget:id":"Naos.Packaging.Domain","nuget:version":"1.0.36"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.05.31/pickles.2.17.0.json","@type":"nuget:PackageDetails","commitId":"b8853431-9b69-487d-ab7a-d7c4dd8e59d0","commitTimeStamp":"2017-11-06T19:05:31.6034108Z","nuget:id":"Pickles","nuget:version":"2.17.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.05.31/pickles.msbuild.2.17.0.json","@type":"nuget:PackageDetails","commitId":"b8853431-9b69-487d-ab7a-d7c4dd8e59d0","commitTimeStamp":"2017-11-06T19:05:31.6034108Z","nuget:id":"Pickles.MSBuild","nuget:version":"2.17.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.02.41/mementofx.persistence.ravendb.2.0.0-pre1.json","@type":"nuget:PackageDetails","commitId":"d65aa625-3473-42ec-8802-2571d2423d40","commitTimeStamp":"2017-11-06T19:02:41.4391839Z","nuget:id":"MementoFX.Persistence.RavenDB","nuget:version":"2.0.0-pre1"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.02.33/mementofx.persistence.ravendb.2.0.0-pre1.json","@type":"nuget:PackageDetails","commitId":"d92b23b3-4b31-4006-944c-1539b9384012","commitTimeStamp":"2017-11-06T19:02:33.1290123Z","nuget:id":"MementoFX.Persistence.RavenDB","nuget:version":"2.0.0-pre1"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.19.00.02/operation.1.1.0.json","@type":"nuget:PackageDetails","commitId":"49dd5128-833f-4c1f-b983-667a1b7dfc31","commitTimeStamp":"2017-11-06T19:00:02.8750219Z","nuget:id":"Operation","nuget:version":"1.1.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.57.16/lightstone.microservice.billable.1.0.6.json","@type":"nuget:PackageDetails","commitId":"a1081394-01af-4f09-9b23-9d54914957b9","commitTimeStamp":"2017-11-06T18:57:16.5089345Z","nuget:id":"Lightstone.MicroService.Billable","nuget:version":"1.0.6"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.57.08/twentytwenty.mvc.0.7.2.json","@type":"nuget:PackageDetails","commitId":"4afbaa09-ccef-4e16-8c6c-b8c9a7934423","commitTimeStamp":"2017-11-06T18:57:08.0868979Z","nuget:id":"TwentyTwenty.Mvc","nuget:version":"0.7.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.57.08/operation.1.1.0.json","@type":"nuget:PackageDetails","commitId":"4afbaa09-ccef-4e16-8c6c-b8c9a7934423","commitTimeStamp":"2017-11-06T18:57:08.0868979Z","nuget:id":"Operation","nuget:version":"1.1.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.54.19/carbon.ci.0.46.5.json","@type":"nuget:PackageDetails","commitId":"fa817ce6-dea4-4dfe-8cee-e4b995583aa9","commitTimeStamp":"2017-11-06T18:54:19.4124314Z","nuget:id":"Carbon.CI","nuget:version":"0.46.5"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.54.11/skiarate.forms.0.5.1-pre.json","@type":"nuget:PackageDetails","commitId":"584a084c-a2d1-47fa-af79-2bef0d29ce56","commitTimeStamp":"2017-11-06T18:54:11.240255Z","nuget:id":"SkiaRate.Forms","nuget:version":"0.5.1-pre"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.54.11/carbon.kms.0.36.0.json","@type":"nuget:PackageDetails","commitId":"584a084c-a2d1-47fa-af79-2bef0d29ce56","commitTimeStamp":"2017-11-06T18:54:11.240255Z","nuget:id":"Carbon.Kms","nuget:version":"0.36.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.54.11/carbon.kms.abstractions.0.36.0.json","@type":"nuget:PackageDetails","commitId":"584a084c-a2d1-47fa-af79-2bef0d29ce56","commitTimeStamp":"2017-11-06T18:54:11.240255Z","nuget:id":"Carbon.Kms.Abstractions","nuget:version":"0.36.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.54.11/carbon.platform.client.0.66.3.json","@type":"nuget:PackageDetails","commitId":"584a084c-a2d1-47fa-af79-2bef0d29ce56","commitTimeStamp":"2017-11-06T18:54:11.240255Z","nuget:id":"Carbon.Platform.Client","nuget:version":"0.66.3"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.54.11/skiarate.0.5.1-pre.json","@type":"nuget:PackageDetails","commitId":"584a084c-a2d1-47fa-af79-2bef0d29ce56","commitTimeStamp":"2017-11-06T18:54:11.240255Z","nuget:id":"SkiaRate","nuget:version":"0.5.1-pre"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.49.24/asyncprocess.net.1.0.0.json","@type":"nuget:PackageDetails","commitId":"03fbde70-12b4-44c7-91ea-a482c0fb4a02","commitTimeStamp":"2017-11-06T18:49:24.1394642Z","nuget:id":"AsyncProcess.Net","nuget:version":"1.0.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.46.54/asyncprocess.net.1.0.0.json","@type":"nuget:PackageDetails","commitId":"60480846-6871-4bec-b35f-dea8be883b98","commitTimeStamp":"2017-11-06T18:46:54.9446581Z","nuget:id":"AsyncProcess.Net","nuget:version":"1.0.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.46.54/bogus.20.0.2.json","@type":"nuget:PackageDetails","commitId":"60480846-6871-4bec-b35f-dea8be883b98","commitTimeStamp":"2017-11-06T18:46:54.9446581Z","nuget:id":"Bogus","nuget:version":"20.0.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.44.27/lightstone.microservice.billable.1.0.6.json","@type":"nuget:PackageDetails","commitId":"3e40407c-a5b6-4f26-8af9-a2009b036252","commitTimeStamp":"2017-11-06T18:44:27.0162517Z","nuget:id":"Lightstone.MicroService.Billable","nuget:version":"1.0.6"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.44.18/lightstone.microservice.billable.1.0.6.json","@type":"nuget:PackageDetails","commitId":"17e453e6-957a-4f17-a85c-05e5875e1a0f","commitTimeStamp":"2017-11-06T18:44:18.5832415Z","nuget:id":"Lightstone.MicroService.Billable","nuget:version":"1.0.6"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.39.13/mysql.simple.3.5.0.json","@type":"nuget:PackageDetails","commitId":"444123aa-d571-4e39-905f-34337be0c889","commitTimeStamp":"2017-11-06T18:39:13.2077861Z","nuget:id":"MySql.Simple","nuget:version":"3.5.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.39.03/mysql.simple.3.5.0.json","@type":"nuget:PackageDetails","commitId":"a80dad57-3eed-4c61-a175-89477f20fe66","commitTimeStamp":"2017-11-06T18:39:03.2413943Z","nuget:id":"MySql.Simple","nuget:version":"3.5.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.39.03/gatewaycontroller.1.0.0.json","@type":"nuget:PackageDetails","commitId":"a80dad57-3eed-4c61-a175-89477f20fe66","commitTimeStamp":"2017-11-06T18:39:03.2413943Z","nuget:id":"GatewayController","nuget:version":"1.0.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.37.04/microsoft.azure.iot.edge.function.0.1.0.json","@type":"nuget:PackageDelete","commitId":"08e2f8a5-fa9e-4788-b437-5b043b618d40","commitTimeStamp":"2017-11-06T18:37:04.8953217Z","nuget:id":"Microsoft.Azure.IoT.Edge.Function","nuget:version":"0.1.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.30/appbrix.factory.0.10.0.json","@type":"nuget:PackageDetails","commitId":"179dbc0b-cec9-4de2-9890-2c655c6ad493","commitTimeStamp":"2017-11-06T18:34:30.6288667Z","nuget:id":"AppBrix.Factory","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.30/appbrix.logging.file.0.10.0.json","@type":"nuget:PackageDetails","commitId":"179dbc0b-cec9-4de2-9890-2c655c6ad493","commitTimeStamp":"2017-11-06T18:34:30.6288667Z","nuget:id":"AppBrix.Logging.File","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.30/appbrix.logging.console.0.10.0.json","@type":"nuget:PackageDetails","commitId":"179dbc0b-cec9-4de2-9890-2c655c6ad493","commitTimeStamp":"2017-11-06T18:34:30.6288667Z","nuget:id":"AppBrix.Logging.Console","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.30/appbrix.time.0.10.0.json","@type":"nuget:PackageDetails","commitId":"179dbc0b-cec9-4de2-9890-2c655c6ad493","commitTimeStamp":"2017-11-06T18:34:30.6288667Z","nuget:id":"AppBrix.Time","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.30/appbrix.text.0.10.0.json","@type":"nuget:PackageDetails","commitId":"179dbc0b-cec9-4de2-9890-2c655c6ad493","commitTimeStamp":"2017-11-06T18:34:30.6288667Z","nuget:id":"AppBrix.Text","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.30/raspberry.io.generalpurpose3.3.1.1.json","@type":"nuget:PackageDetails","commitId":"179dbc0b-cec9-4de2-9890-2c655c6ad493","commitTimeStamp":"2017-11-06T18:34:30.6288667Z","nuget:id":"Raspberry.IO.GeneralPurpose3","nuget:version":"3.1.1"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.30/appbrix.logging.0.10.0.json","@type":"nuget:PackageDetails","commitId":"179dbc0b-cec9-4de2-9890-2c655c6ad493","commitTimeStamp":"2017-11-06T18:34:30.6288667Z","nuget:id":"AppBrix.Logging","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.30/appbrix.web.server.0.10.0.json","@type":"nuget:PackageDetails","commitId":"179dbc0b-cec9-4de2-9890-2c655c6ad493","commitTimeStamp":"2017-11-06T18:34:30.6288667Z","nuget:id":"AppBrix.Web.Server","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.30/appbrix.events.schedule.timer.0.10.0.json","@type":"nuget:PackageDetails","commitId":"179dbc0b-cec9-4de2-9890-2c655c6ad493","commitTimeStamp":"2017-11-06T18:34:30.6288667Z","nuget:id":"AppBrix.Events.Schedule.Timer","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.30/appbrix.events.schedule.cron.0.10.0.json","@type":"nuget:PackageDetails","commitId":"179dbc0b-cec9-4de2-9890-2c655c6ad493","commitTimeStamp":"2017-11-06T18:34:30.6288667Z","nuget:id":"AppBrix.Events.Schedule.Cron","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.30/raspberry.io.interintegratedcircuit3.3.1.1.json","@type":"nuget:PackageDetails","commitId":"179dbc0b-cec9-4de2-9890-2c655c6ad493","commitTimeStamp":"2017-11-06T18:34:30.6288667Z","nuget:id":"Raspberry.IO.InterIntegratedCircuit3","nuget:version":"3.1.1"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.30/appbrix.web.client.0.10.0.json","@type":"nuget:PackageDetails","commitId":"179dbc0b-cec9-4de2-9890-2c655c6ad493","commitTimeStamp":"2017-11-06T18:34:30.6288667Z","nuget:id":"AppBrix.Web.Client","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.configuration.files.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Configuration.Files","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/raspberry.io.serialperipheralinterface3.3.1.1.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"Raspberry.IO.SerialPeripheralInterface3","nuget:version":"3.1.1"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.data.migration.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Data.Migration","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/lifeimage-agent.1.0.18.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"lifeimage-agent","nuget:version":"1.0.18"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.data.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Data","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/syncsoft.cqrs.1.2.3.1.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"SyncSoft.CQRS","nuget:version":"1.2.3.1"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.events.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Events","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.data.sqlite.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Data.Sqlite","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.configuration.json.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Configuration.Json","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.cloning.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Cloning","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.data.sqlserver.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Data.SqlServer","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.caching.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Caching","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.events.schedule.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Events.Schedule","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.data.inmemory.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Data.InMemory","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.container.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Container","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.events.async.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Events.Async","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.configuration.yaml.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Configuration.Yaml","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.configuration.memory.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Configuration.Memory","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.caching.memory.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Caching.Memory","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.34.20/appbrix.utils.0.10.0.json","@type":"nuget:PackageDetails","commitId":"9588120e-1f6b-4fc8-8b02-eed98536fd43","commitTimeStamp":"2017-11-06T18:34:20.5021327Z","nuget:id":"AppBrix.Utils","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.32.18/iot.edge.function.0.1.0.json","@type":"nuget:PackageDelete","commitId":"c5d7a6b9-07e5-4273-9835-e6b52230e295","commitTimeStamp":"2017-11-06T18:32:18.2707545Z","nuget:id":"IoT.Edge.Function","nuget:version":"0.1.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.30.11/iot.edge.module.0.1.0.json","@type":"nuget:PackageDelete","commitId":"b82b18a1-5918-4625-8cb5-ca5689f371eb","commitTimeStamp":"2017-11-06T18:30:11.5733843Z","nuget:id":"IoT.Edge.Module","nuget:version":"0.1.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.28.07/kitaboosdk.1.0.0.2.json","@type":"nuget:PackageDelete","commitId":"8059ad1c-ce1c-4fb6-8d9b-bd897a65acd1","commitTimeStamp":"2017-11-06T18:28:07.3663249Z","nuget:id":"KitabooSDK","nuget:version":"1.0.0.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.25.40/appbrix.configuration.0.10.0.json","@type":"nuget:PackageDetails","commitId":"94357105-a901-4eb4-a619-700ffcb1c678","commitTimeStamp":"2017-11-06T18:25:40.6940307Z","nuget:id":"AppBrix.Configuration","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.25.31/raspberry.system3.3.1.1.json","@type":"nuget:PackageDetails","commitId":"8205fa6a-4015-473c-aaaf-44dd86c062cb","commitTimeStamp":"2017-11-06T18:25:31.6127782Z","nuget:id":"Raspberry.System3","nuget:version":"3.1.1"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.25.31/appbrix.0.10.0.json","@type":"nuget:PackageDetails","commitId":"8205fa6a-4015-473c-aaaf-44dd86c062cb","commitTimeStamp":"2017-11-06T18:25:31.6127782Z","nuget:id":"AppBrix","nuget:version":"0.10.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.23.10/lykke.wampsharp.1.0.2.json","@type":"nuget:PackageDetails","commitId":"d60c4c60-f4ff-4a5c-a800-1266047c9f9f","commitTimeStamp":"2017-11-06T18:23:10.7115861Z","nuget:id":"Lykke.WampSharp","nuget:version":"1.0.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.23.00/lykke.wampsharp.websockets.1.0.2.json","@type":"nuget:PackageDetails","commitId":"38bef28c-4676-483e-9550-a152f82967b4","commitTimeStamp":"2017-11-06T18:23:00.8808904Z","nuget:id":"Lykke.WampSharp.WebSockets","nuget:version":"1.0.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.22.50/lykke.wampsharp.default.client.1.0.2.json","@type":"nuget:PackageDetails","commitId":"2d08d48c-8c1e-46d1-96df-41f841191bab","commitTimeStamp":"2017-11-06T18:22:50.8080398Z","nuget:id":"Lykke.WampSharp.Default.Client","nuget:version":"1.0.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.22.50/lykke.wampsharp.newtonsoftmsgpack.1.0.2.json","@type":"nuget:PackageDetails","commitId":"2d08d48c-8c1e-46d1-96df-41f841191bab","commitTimeStamp":"2017-11-06T18:22:50.8080398Z","nuget:id":"Lykke.WampSharp.NewtonsoftMsgpack","nuget:version":"1.0.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.22.50/lykke.wampsharp.newtonsoftjson.1.0.2.json","@type":"nuget:PackageDetails","commitId":"2d08d48c-8c1e-46d1-96df-41f841191bab","commitTimeStamp":"2017-11-06T18:22:50.8080398Z","nuget:id":"Lykke.WampSharp.NewtonsoftJson","nuget:version":"1.0.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.22.50/lykke.wampsharp.websocket4net.1.0.2.json","@type":"nuget:PackageDetails","commitId":"2d08d48c-8c1e-46d1-96df-41f841191bab","commitTimeStamp":"2017-11-06T18:22:50.8080398Z","nuget:id":"Lykke.WampSharp.WebSocket4Net","nuget:version":"1.0.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.22.50/ah.framework.dataaccesslayer.3.0.0.json","@type":"nuget:PackageDetails","commitId":"2d08d48c-8c1e-46d1-96df-41f841191bab","commitTimeStamp":"2017-11-06T18:22:50.8080398Z","nuget:id":"Ah.Framework.DataAccessLayer","nuget:version":"3.0.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.22.50/lykke.wampsharp.aspnetcore.websockets.server.1.0.2.json","@type":"nuget:PackageDetails","commitId":"2d08d48c-8c1e-46d1-96df-41f841191bab","commitTimeStamp":"2017-11-06T18:22:50.8080398Z","nuget:id":"Lykke.WampSharp.AspNetCore.WebSockets.Server","nuget:version":"1.0.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.18.10/owlcarousel.1.3.3.json","@type":"nuget:PackageDetails","commitId":"890d2ee8-7b9b-4843-b1fe-85939614acd0","commitTimeStamp":"2017-11-06T18:18:10.2021165Z","nuget:id":"OwlCarousel","nuget:version":"1.3.3"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.18.01/winton.extensions.configuration.consul.1.1.0-master0003.json","@type":"nuget:PackageDetails","commitId":"4c95a935-4b0e-4650-8623-ba38b2992ba5","commitTimeStamp":"2017-11-06T18:18:01.6394392Z","nuget:id":"Winton.Extensions.Configuration.Consul","nuget:version":"1.1.0-master0003"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.15.34/moq.4.7.145.json","@type":"nuget:PackageDetails","commitId":"eec47a84-4488-4586-b7a7-371d4a5b7804","commitTimeStamp":"2017-11-06T18:15:34.0631069Z","nuget:id":"Moq","nuget:version":"4.7.145"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.15.34/claytondus.amazonmws.fbaoutbound.0.2.0.json","@type":"nuget:PackageDetails","commitId":"eec47a84-4488-4586-b7a7-371d4a5b7804","commitTimeStamp":"2017-11-06T18:15:34.0631069Z","nuget:id":"Claytondus.AmazonMWS.FbaOutbound","nuget:version":"0.2.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.13.05/walkingtec.mvvm.mvc.1.1.2.json","@type":"nuget:PackageDetails","commitId":"4fda0f59-5d64-4d89-971a-5025a170e014","commitTimeStamp":"2017-11-06T18:13:05.3692258Z","nuget:id":"WalkingTec.Mvvm.Mvc","nuget:version":"1.1.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.13.05/sentrydotnet.aspnetcore.1.1.0.json","@type":"nuget:PackageDetails","commitId":"4fda0f59-5d64-4d89-971a-5025a170e014","commitTimeStamp":"2017-11-06T18:13:05.3692258Z","nuget:id":"SentryDotNet.AspNetCore","nuget:version":"1.1.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.13.05/walkingtec.mvvm.taghelpers.layui.1.1.2.json","@type":"nuget:PackageDetails","commitId":"4fda0f59-5d64-4d89-971a-5025a170e014","commitTimeStamp":"2017-11-06T18:13:05.3692258Z","nuget:id":"WalkingTec.Mvvm.TagHelpers.LayUI","nuget:version":"1.1.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.13.05/walkingtec.mvvm.core.1.1.2.json","@type":"nuget:PackageDetails","commitId":"4fda0f59-5d64-4d89-971a-5025a170e014","commitTimeStamp":"2017-11-06T18:13:05.3692258Z","nuget:id":"WalkingTec.Mvvm.Core","nuget:version":"1.1.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/peachpie.library.pdo.sqlite.0.8.0-ci00457.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"Peachpie.Library.PDO.Sqlite","nuget:version":"0.8.0-CI00457"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/sentrydotnet.1.1.0.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"SentryDotNet","nuget:version":"1.1.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/peachpie.library.mysql.0.8.0-ci00457.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"Peachpie.Library.MySql","nuget:version":"0.8.0-CI00457"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/peachpie.library.pdo.firebird.0.8.0-ci00457.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"Peachpie.Library.PDO.Firebird","nuget:version":"0.8.0-CI00457"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/askmethat.xforms.controls.1.1.1.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"Askmethat.XForms.Controls","nuget:version":"1.1.1"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/peachpie.library.pdo.sqlsrv.0.8.0-ci00457.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"Peachpie.Library.PDO.SqlSrv","nuget:version":"0.8.0-CI00457"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/peachpie.requesthandler.0.8.0-ci00457.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"Peachpie.RequestHandler","nuget:version":"0.8.0-CI00457"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/walkingtec.mvvm.core.1.1.2.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"WalkingTec.Mvvm.Core","nuget:version":"1.1.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/peachpie.library.pdo.pgsql.0.8.0-ci00457.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"Peachpie.Library.PDO.PgSQL","nuget:version":"0.8.0-CI00457"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/peachpie.net.sdk.0.8.0-ci00457.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"Peachpie.NET.Sdk","nuget:version":"0.8.0-CI00457"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/sentrydotnet.aspnetcore.1.1.0.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"SentryDotNet.AspNetCore","nuget:version":"1.1.0"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/peachpie.library.pdo.mysql.0.8.0-ci00457.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"Peachpie.Library.PDO.MySQL","nuget:version":"0.8.0-CI00457"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/walkingtec.mvvm.taghelpers.layui.1.1.2.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"WalkingTec.Mvvm.TagHelpers.LayUI","nuget:version":"1.1.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/peachpie.library.pdo.0.8.0-ci00457.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"Peachpie.Library.PDO","nuget:version":"0.8.0-CI00457"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/walkingtec.mvvm.mvc.1.1.2.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"WalkingTec.Mvvm.Mvc","nuget:version":"1.1.2"},{"@id":"https://api.nuget.org/v3/catalog0/data/2017.11.06.18.12.56/peachpie.library.scripting.0.8.0-ci00457.json","@type":"nuget:PackageDetails","commitId":"355d531a-f3ac-43e4-8355-357ad8aa0456","commitTimeStamp":"2017-11-06T18:12:56.3806831Z","nuget:id":"Peachpie.Library.Scripting","nuget:version":"0.8.0-CI00457"}],"@context":{"@vocab":"http://schema.nuget.org/catalog#","nuget":"http://schema.nuget.org/schema#","items":{"@id":"item","@container":"@set"},"parent":{"@type":"@id"},"commitTimeStamp":{"@type":"http://www.w3.org/2001/XMLSchema#dateTime"},"nuget:lastCreated":{"@type":"http://www.w3.org/2001/XMLSchema#dateTime"},"nuget:lastEdited":{"@type":"http://www.w3.org/2001/XMLSchema#dateTime"},"nuget:lastDeleted":{"@type":"http://www.w3.org/2001/XMLSchema#dateTime"}}} + + + https://api.nuget.org/v3/catalog0/page2944.json + + + { + "@id": "https://api.nuget.org/v3/catalog0/data/2017.11.06.19.29.03/microsoft.azure.iot.edge.function.0.3.0.json", + "@type": [ + "PackageDelete", + "catalog:Permalink" + ], + "catalog:commitId": "b429c4e3-5127-4430-9cc8-74927c9c3886", + "catalog:commitTimeStamp": "2017-11-06T19:29:03.6198426Z", + "id": "Microsoft.Azure.IoT.Edge.Function", + "originalId": "Microsoft.Azure.IoT.Edge.Function", + "published": "2017-11-06T19:27:45.3684766Z", + "version": "0.3.0", + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "details": "catalog:details", + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + }, + "published": { + "@type": "xsd:dateTime" + }, + "categories": { + "@container": "@set" + }, + "entries": { + "@container": "@set" + }, + "links": { + "@container": "@set" + }, + "tags": { + "@container": "@set" + }, + "packageContent": { + "@type": "@id" + } + } +} + + + https://api.nuget.org/v3/catalog0/data/2017.11.06.19.29.03/microsoft.azure.iot.edge.function.0.3.0.json + + + { + "@id": "https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.container.1.3.2.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "Darrell Tunnell", + "catalog:commitId": "f241ce46-35ba-44c2-bd72-790eb44539a5", + "catalog:commitTimeStamp": "2017-11-06T22:07:49.3270578Z", + "created": "2017-11-06T22:06:53.64Z", + "description": "Container support, for the dotnettency Mutlitenancy library for dotnet standard compatible applications.", + "id": "Dotnettency.Container", + "isPrerelease": false, + "lastEdited": "0001-01-01T00:00:00Z", + "listed": true, + "packageHash": "ig2qG5u7ua4mGifVb1PA1HyDt0pkKWsXzR1t4fBrsOP3ZDRm/1FeNho8G4QD4Pw/bGE+EdOpRe5+0MD+ANw4uw==", + "packageHashAlgorithm": "SHA512", + "packageSize": 7945, + "projectUrl": "https://github.com/dazinator/Dotnettency", + "published": "2017-11-06T22:06:53.64Z", + "requireLicenseAcceptance": false, + "verbatimVersion": "1.3.2", + "version": "1.3.2", + "dependencyGroups": [ + { + "@id": "https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.container.1.3.2.json#dependencygroup/.netstandard1.3", + "@type": "PackageDependencyGroup", + "dependencies": [ + { + "@id": "https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.container.1.3.2.json#dependencygroup/.netstandard1.3/dotnettency", + "@type": "PackageDependency", + "id": "Dotnettency", + "range": "[1.3.2, )" + }, + { + "@id": "https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.container.1.3.2.json#dependencygroup/.netstandard1.3/netstandard.library", + "@type": "PackageDependency", + "id": "NETStandard.Library", + "range": "[1.6.1, )" + }, + { + "@id": "https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.container.1.3.2.json#dependencygroup/.netstandard1.3/microsoft.extensions.dependencyinjection", + "@type": "PackageDependency", + "id": "Microsoft.Extensions.DependencyInjection", + "range": "[1.0.0, )" + } + ], + "targetFramework": ".NETStandard1.3" + } + ], + "packageEntries": [ + { + "@id": "https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.container.1.3.2.json#Dotnettency.Container.nuspec", + "@type": "PackageEntry", + "compressedLength": 478, + "fullName": "Dotnettency.Container.nuspec", + "length": 1016, + "name": "Dotnettency.Container.nuspec" + }, + { + "@id": "https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.container.1.3.2.json#lib/netstandard1.3/Dotnettency.Container.dll", + "@type": "PackageEntry", + "compressedLength": 5751, + "fullName": "lib/netstandard1.3/Dotnettency.Container.dll", + "length": 13312, + "name": "Dotnettency.Container.dll" + } + ], + "tags": [ + "Multitenancy", + "tenant", + "container" + ], + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + } + } +} + + + https://api.nuget.org/v3/catalog0/data/2017.11.06.22.07.49/dotnettency.container.1.3.2.json + + + { + "@id": "https://api.nuget.org/v3/catalog0/data/2015.02.01.06.22.45/antixss.4.0.1.json", + "@type": [ + "PackageDetails", + "catalog:Permalink" + ], + "authors": "Microsoft", + "catalog:commitId": "b3f4fc8a-7522-42a3-8fee-a91d5488c0b1", + "catalog:commitTimeStamp": "2015-02-01T06:22:45.8488496Z", + "created": "2011-01-07T07:49:50.307Z", + "description": "AntiXSS is an encoding library which uses a safe list approach to encoding. It provides Html, XML, Url, Form, LDAP, CSS, JScript and VBScript encoding methods to allow \nyou to avoid Cross Site Scripting attacks. This library is part of the Microsoft SDL tools.", + "id": "AntiXSS", + "isPrerelease": false, + "language": "en-US", + "lastEdited": "0001-01-01T00:00:00Z", + "licenseUrl": "http://wpl.codeplex.com/license", + "packageHash": "Yim7Vde4dMtYF2mASw881JHKqs/TJ70O8QSIXAETYjznpKGxOXx46nbKbuldU9OgFqHcXabMZJWu8MVU4/uHOg==", + "packageHashAlgorithm": "SHA512", + "packageSize": 329780, + "published": "1900-01-01T00:00:00Z", + "requireLicenseAcceptance": true, + "summary": "AntiXSS is an encoding library which uses a safe list approach to encoding. It provides Html, XML, Url, Form, LDAP, CSS, JScript and VBScript encoding methods to allow \nyou to avoid Cross Site Scripting attacks. This library is part of the Microsoft SDL tools.", + "version": "4.0.1", + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "catalog": "http://schema.nuget.org/catalog#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dependencies": { + "@id": "dependency", + "@container": "@set" + }, + "dependencyGroups": { + "@id": "dependencyGroup", + "@container": "@set" + }, + "packageEntries": { + "@id": "packageEntry", + "@container": "@set" + }, + "packageTypes": { + "@id": "packageType", + "@container": "@set" + }, + "supportedFrameworks": { + "@id": "supportedFramework", + "@container": "@set" + }, + "tags": { + "@id": "tag", + "@container": "@set" + }, + "published": { + "@type": "xsd:dateTime" + }, + "created": { + "@type": "xsd:dateTime" + }, + "lastEdited": { + "@type": "xsd:dateTime" + }, + "catalog:commitTimeStamp": { + "@type": "xsd:dateTime" + } + } +} + + + https://api.nuget.org/v3/catalog0/data/2015.02.01.06.22.45/antixss.4.0.1.json + + + {"@id":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/index.json","@type":["catalog:CatalogRoot","PackageRegistration","catalog:Permalink"],"commitId":"bef2767e-f5ae-4713-b5fc-510945bacdf9","commitTimeStamp":"2018-11-26T19:53:07.9393299Z","count":1,"items":[{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/index.json#page/0.1.1/0.3.1","@type":"catalog:CatalogPage","commitId":"bef2767e-f5ae-4713-b5fc-510945bacdf9","commitTimeStamp":"2018-11-26T19:53:07.9393299Z","count":4,"items":[{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/0.1.1.json","@type":"Package","commitId":"bef2767e-f5ae-4713-b5fc-510945bacdf9","commitTimeStamp":"2018-11-26T19:53:07.9393299Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.11.13.04.43.04/microbuild.core.0.1.1.json","@type":"PackageDetails","authors":"Microsoft","description":"MicroBuild bootstrapper package which wires up build targets that load and execute other MicroBuild plugins during the build","iconUrl":"","id":"MicroBuild.Core","language":"","licenseUrl":"","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/microbuild.core/0.1.1/microbuild.core.0.1.1.nupkg","projectUrl":"","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":false,"summary":"","tags":["MicroBuild"],"title":"","version":"0.1.1"},"packageContent":"https://api.nuget.org/v3-flatcontainer/microbuild.core/0.1.1/microbuild.core.0.1.1.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/0.2.0.json","@type":"Package","commitId":"bef2767e-f5ae-4713-b5fc-510945bacdf9","commitTimeStamp":"2018-11-26T19:53:07.9393299Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.11.13.04.43.04/microbuild.core.0.2.0.json","@type":"PackageDetails","authors":"Microsoft","description":"MicroBuild bootstrapper package which wires up build targets that load and execute other MicroBuild plugins during the build","iconUrl":"","id":"MicroBuild.Core","language":"","licenseUrl":"http://microsoft.mit-license.org/","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/microbuild.core/0.2.0/microbuild.core.0.2.0.nupkg","projectUrl":"","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":false,"summary":"","tags":["MicroBuild"],"title":"","version":"0.2.0"},"packageContent":"https://api.nuget.org/v3-flatcontainer/microbuild.core/0.2.0/microbuild.core.0.2.0.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/0.3.0.json","@type":"Package","commitId":"bef2767e-f5ae-4713-b5fc-510945bacdf9","commitTimeStamp":"2018-11-26T19:53:07.9393299Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.11.26.19.52.28/microbuild.core.0.3.0.json","@type":"PackageDetails","authors":"Microsoft","description":"MicroBuild bootstrapper package which wires up build targets that load and execute other MicroBuild plugins during the build","iconUrl":"","id":"MicroBuild.Core","language":"","licenseUrl":"http://microsoft.mit-license.org/","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/microbuild.core/0.3.0/microbuild.core.0.3.0.nupkg","projectUrl":"","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":false,"summary":"","tags":["MicroBuild"],"title":"","version":"0.3.0"},"packageContent":"https://api.nuget.org/v3-flatcontainer/microbuild.core/0.3.0/microbuild.core.0.3.0.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/0.3.1.json","@type":"Package","commitId":"bef2767e-f5ae-4713-b5fc-510945bacdf9","commitTimeStamp":"2018-11-26T19:53:07.9393299Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.11.26.19.49.07/microbuild.core.0.3.1.json","@type":"PackageDetails","authors":"Microsoft","description":"MicroBuild bootstrapper package which wires up build targets that load and execute other MicroBuild plugins during the build","iconUrl":"https://github.com/MicrosoftDocs/Microbuild.Core/blob/master/extension-icon.png?raw=true","id":"MicroBuild.Core","language":"en-US","licenseUrl":"http://microsoft.mit-license.org/","listed":true,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/microbuild.core/0.3.1/microbuild.core.0.3.1.nupkg","projectUrl":"https://aka.ms/microbuild.package","published":"2018-11-13T04:26:56.943+00:00","requireLicenseAcceptance":true,"summary":"","tags":["MicroBuild"],"title":"","version":"0.3.1"},"packageContent":"https://api.nuget.org/v3-flatcontainer/microbuild.core/0.3.1/microbuild.core.0.3.1.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/index.json"}],"parent":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/index.json","lower":"0.1.1","upper":"0.3.1"}],"@context":{"@vocab":"http://schema.nuget.org/schema#","catalog":"http://schema.nuget.org/catalog#","xsd":"http://www.w3.org/2001/XMLSchema#","items":{"@id":"catalog:item","@container":"@set"},"commitTimeStamp":{"@id":"catalog:commitTimeStamp","@type":"xsd:dateTime"},"commitId":{"@id":"catalog:commitId"},"count":{"@id":"catalog:count"},"parent":{"@id":"catalog:parent","@type":"@id"},"tags":{"@container":"@set","@id":"tag"},"packageTargetFrameworks":{"@container":"@set","@id":"packageTargetFramework"},"dependencyGroups":{"@container":"@set","@id":"dependencyGroup"},"dependencies":{"@container":"@set","@id":"dependency"},"packageContent":{"@type":"@id"},"published":{"@type":"xsd:dateTime"},"registration":{"@type":"@id"}}} + + + https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/index.json + + + {"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json","@type":["catalog:CatalogRoot","PackageRegistration","catalog:Permalink"],"commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":19,"items":[{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/1.0.728-unstable/1.0.965-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"1.0.728-Unstable","upper":"1.0.965-Unstable"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/1.0.989/1.2.2064-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"1.0.989","upper":"1.2.2064-Unstable"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/1.2.2065-unstable/1.2.2132-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"1.2.2065-Unstable","upper":"1.2.2132-Unstable"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/1.2.2133-unstable/2.0.2235-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"1.2.2133-Unstable","upper":"2.0.2235-Unstable"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/2.0.2236-unstable/2.5.2605-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"2.0.2236-Unstable","upper":"2.5.2605-Unstable"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/2.5.2606-unstable/2.5.2678-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"2.5.2606-Unstable","upper":"2.5.2678-Unstable"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/2.5.2679-unstable/2.5.2763-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"2.5.2679-Unstable","upper":"2.5.2763-Unstable"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/2.5.2764-unstable/2.5.2870-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"2.5.2764-Unstable","upper":"2.5.2870-Unstable"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/2.5.2870/2.5.2939.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"2.5.2870","upper":"2.5.2939"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/2.5.2940-unstable/2.5.25031.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"2.5.2940-Unstable","upper":"2.5.25031"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/2.5.25032/3.0.3590-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"2.5.25032","upper":"3.0.3590-Unstable"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/3.0.3591-unstable/3.0.3666-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"3.0.3591-Unstable","upper":"3.0.3666-Unstable"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/3.0.3667-unstable/3.0.3746-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"3.0.3667-Unstable","upper":"3.0.3746-Unstable"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/3.0.3747-unstable/3.0.3831-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"3.0.3747-Unstable","upper":"3.0.3831-Unstable"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/3.0.3832-unstable/3.0.30064-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"3.0.3832-Unstable","upper":"3.0.30064-Unstable"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/3.0.30065-unstable/3.0.30179.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"3.0.30065-Unstable","upper":"3.0.30179"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/3.0.30180-hotfix/3.5.6-patch-35248.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"3.0.30180-Hotfix","upper":"3.5.6-patch-35248"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/3.5.6-patch-35249/3.5.35077-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":64,"lower":"3.5.6-patch-35249","upper":"3.5.35077-Unstable"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/3.5.35078-unstable/3.5.35142-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":55,"lower":"3.5.35078-Unstable","upper":"3.5.35142-Unstable"}],"@context":{"@vocab":"http://schema.nuget.org/schema#","catalog":"http://schema.nuget.org/catalog#","xsd":"http://www.w3.org/2001/XMLSchema#","items":{"@id":"catalog:item","@container":"@set"},"commitTimeStamp":{"@id":"catalog:commitTimeStamp","@type":"xsd:dateTime"},"commitId":{"@id":"catalog:commitId"},"count":{"@id":"catalog:count"},"parent":{"@id":"catalog:parent","@type":"@id"},"tags":{"@container":"@set","@id":"tag"},"packageTargetFrameworks":{"@container":"@set","@id":"packageTargetFramework"},"dependencyGroups":{"@container":"@set","@id":"dependencyGroup"},"dependencies":{"@container":"@set","@id":"dependency"},"packageContent":{"@type":"@id"},"published":{"@type":"xsd:dateTime"},"registration":{"@type":"@id"}}} + + + https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json + + + {"@id":"https://api.nuget.org/v3/registration3-gz-semver2/newtonsoft.json/12.0.1.json","@type":["Package","http://schema.nuget.org/catalog#Permalink"],"catalogEntry":"https://api.nuget.org/v3/catalog0/data/2018.11.27.18.15.55/newtonsoft.json.12.0.1.json","listed":true,"packageContent":"https://api.nuget.org/v3-flatcontainer/newtonsoft.json/12.0.1/newtonsoft.json.12.0.1.nupkg","published":"2018-11-27T18:11:37.08+00:00","registration":"https://api.nuget.org/v3/registration3-gz-semver2/newtonsoft.json/index.json","@context":{"@vocab":"http://schema.nuget.org/schema#","xsd":"http://www.w3.org/2001/XMLSchema#","catalogEntry":{"@type":"@id"},"registration":{"@type":"@id"},"packageContent":{"@type":"@id"},"published":{"@type":"xsd:dateTime"}}} + + + https://api.nuget.org/v3/registration3-gz-semver2/newtonsoft.json/12.0.1.json + + + {"@id":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/0.1.1.json","@type":["Package","http://schema.nuget.org/catalog#Permalink"],"catalogEntry":"https://api.nuget.org/v3/catalog0/data/2018.11.13.04.43.04/microbuild.core.0.1.1.json","listed":false,"packageContent":"https://api.nuget.org/v3-flatcontainer/microbuild.core/0.1.1/microbuild.core.0.1.1.nupkg","published":"1900-01-01T00:00:00+00:00","registration":"https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/index.json","@context":{"@vocab":"http://schema.nuget.org/schema#","xsd":"http://www.w3.org/2001/XMLSchema#","catalogEntry":{"@type":"@id"},"registration":{"@type":"@id"},"packageContent":{"@type":"@id"},"published":{"@type":"xsd:dateTime"}}} + + + https://api.nuget.org/v3/registration3-gz-semver2/microbuild.core/0.1.1.json + + + {"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/3.5.35078-unstable/3.5.35142-unstable.json","@type":"catalog:CatalogPage","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","count":55,"lower":"3.5.35078-Unstable","parent":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json","upper":"3.5.35142-Unstable","items":[{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35078-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.21/ravendb.database.3.5.35078-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.21/ravendb.database.3.5.35078-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.21/ravendb.database.3.5.35078-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35078-Unstable, 3.5.35078-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35078-unstable/ravendb.database.3.5.35078-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["database","document","nosql","ravendb","raven"],"title":"RavenDB Database","version":"3.5.35078-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35078-unstable/ravendb.database.3.5.35078-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35079-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.21/ravendb.database.3.5.35079-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.21/ravendb.database.3.5.35079-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.21/ravendb.database.3.5.35079-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35079-Unstable, 3.5.35079-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35079-unstable/ravendb.database.3.5.35079-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","document","raven","nosql","database"],"title":"RavenDB Database","version":"3.5.35079-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35079-unstable/ravendb.database.3.5.35079-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35080-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.21/ravendb.database.3.5.35080-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.21/ravendb.database.3.5.35080-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.21/ravendb.database.3.5.35080-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35080-Unstable, 3.5.35080-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35080-unstable/ravendb.database.3.5.35080-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","nosql","database","raven","document"],"title":"RavenDB Database","version":"3.5.35080-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35080-unstable/ravendb.database.3.5.35080-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35081-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.59/ravendb.database.3.5.35081-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.59/ravendb.database.3.5.35081-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.59/ravendb.database.3.5.35081-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35081-Unstable, 3.5.35081-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35081-unstable/ravendb.database.3.5.35081-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["database","nosql","document","ravendb","raven"],"title":"RavenDB Database","version":"3.5.35081-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35081-unstable/ravendb.database.3.5.35081-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35082-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.41/ravendb.database.3.5.35082-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.41/ravendb.database.3.5.35082-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.41/ravendb.database.3.5.35082-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35082-Unstable, 3.5.35082-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35082-unstable/ravendb.database.3.5.35082-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["nosql","document","ravendb","database","raven"],"title":"RavenDB Database","version":"3.5.35082-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35082-unstable/ravendb.database.3.5.35082-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35083-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.41/ravendb.database.3.5.35083-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.41/ravendb.database.3.5.35083-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.41/ravendb.database.3.5.35083-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35083-Unstable, 3.5.35083-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35083-unstable/ravendb.database.3.5.35083-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["raven","ravendb","document","nosql","database"],"title":"RavenDB Database","version":"3.5.35083-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35083-unstable/ravendb.database.3.5.35083-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35084-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.21/ravendb.database.3.5.35084-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.21/ravendb.database.3.5.35084-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.16.21/ravendb.database.3.5.35084-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35084-Unstable, 3.5.35084-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35084-unstable/ravendb.database.3.5.35084-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","document","raven","database","nosql"],"title":"RavenDB Database","version":"3.5.35084-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35084-unstable/ravendb.database.3.5.35084-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35085-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.03/ravendb.database.3.5.35085-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.03/ravendb.database.3.5.35085-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.03/ravendb.database.3.5.35085-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35085-Unstable, 3.5.35085-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35085-unstable/ravendb.database.3.5.35085-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","database","document","raven","nosql"],"title":"RavenDB Database","version":"3.5.35085-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35085-unstable/ravendb.database.3.5.35085-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35086-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.03/ravendb.database.3.5.35086-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.03/ravendb.database.3.5.35086-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.03/ravendb.database.3.5.35086-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35086-Unstable, 3.5.35086-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35086-unstable/ravendb.database.3.5.35086-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["nosql","raven","ravendb","database","document"],"title":"RavenDB Database","version":"3.5.35086-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35086-unstable/ravendb.database.3.5.35086-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35087-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.41/ravendb.database.3.5.35087-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.41/ravendb.database.3.5.35087-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.41/ravendb.database.3.5.35087-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35087-Unstable, 3.5.35087-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35087-unstable/ravendb.database.3.5.35087-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","raven","document","nosql","database"],"title":"RavenDB Database","version":"3.5.35087-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35087-unstable/ravendb.database.3.5.35087-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35089-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.48/ravendb.database.3.5.35089-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.48/ravendb.database.3.5.35089-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.48/ravendb.database.3.5.35089-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35089-Unstable, 3.5.35089-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35089-unstable/ravendb.database.3.5.35089-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["nosql","database","document","raven","ravendb"],"title":"RavenDB Database","version":"3.5.35089-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35089-unstable/ravendb.database.3.5.35089-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35090-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.41/ravendb.database.3.5.35090-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.41/ravendb.database.3.5.35090-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.15.41/ravendb.database.3.5.35090-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35090-Unstable, 3.5.35090-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35090-unstable/ravendb.database.3.5.35090-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["raven","nosql","ravendb","database","document"],"title":"RavenDB Database","version":"3.5.35090-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35090-unstable/ravendb.database.3.5.35090-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35091-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.42/ravendb.database.3.5.35091-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.42/ravendb.database.3.5.35091-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.42/ravendb.database.3.5.35091-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35091-Unstable, 3.5.35091-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35091-unstable/ravendb.database.3.5.35091-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["document","ravendb","database","raven","nosql"],"title":"RavenDB Database","version":"3.5.35091-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35091-unstable/ravendb.database.3.5.35091-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35092-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.04/ravendb.database.3.5.35092-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.04/ravendb.database.3.5.35092-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.04/ravendb.database.3.5.35092-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35092-Unstable, 3.5.35092-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35092-unstable/ravendb.database.3.5.35092-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["database","ravendb","document","raven","nosql"],"title":"RavenDB Database","version":"3.5.35092-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35092-unstable/ravendb.database.3.5.35092-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35093-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.04/ravendb.database.3.5.35093-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.04/ravendb.database.3.5.35093-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.04/ravendb.database.3.5.35093-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35093-Unstable, 3.5.35093-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35093-unstable/ravendb.database.3.5.35093-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","raven","nosql","document","database"],"title":"RavenDB Database","version":"3.5.35093-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35093-unstable/ravendb.database.3.5.35093-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35095-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.04/ravendb.database.3.5.35095-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.04/ravendb.database.3.5.35095-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.04/ravendb.database.3.5.35095-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35095-Unstable, 3.5.35095-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35095-unstable/ravendb.database.3.5.35095-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["raven","ravendb","nosql","database","document"],"title":"RavenDB Database","version":"3.5.35095-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35095-unstable/ravendb.database.3.5.35095-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35096-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.56/ravendb.database.3.5.35096-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.56/ravendb.database.3.5.35096-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.56/ravendb.database.3.5.35096-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35096-Unstable, 3.5.35096-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35096-unstable/ravendb.database.3.5.35096-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["raven","nosql","document","ravendb","database"],"title":"RavenDB Database","version":"3.5.35096-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35096-unstable/ravendb.database.3.5.35096-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35097-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.13.24/ravendb.database.3.5.35097-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.13.24/ravendb.database.3.5.35097-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.13.24/ravendb.database.3.5.35097-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35097-Unstable, 3.5.35097-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35097-unstable/ravendb.database.3.5.35097-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["database","nosql","ravendb","raven","document"],"title":"RavenDB Database","version":"3.5.35097-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35097-unstable/ravendb.database.3.5.35097-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35098-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.13.24/ravendb.database.3.5.35098-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.13.24/ravendb.database.3.5.35098-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.13.24/ravendb.database.3.5.35098-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35098-Unstable, 3.5.35098-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35098-unstable/ravendb.database.3.5.35098-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["nosql","ravendb","raven","document","database"],"title":"RavenDB Database","version":"3.5.35098-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35098-unstable/ravendb.database.3.5.35098-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35100-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.13.24/ravendb.database.3.5.35100-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.13.24/ravendb.database.3.5.35100-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.13.24/ravendb.database.3.5.35100-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35100-Unstable, 3.5.35100-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35100-unstable/ravendb.database.3.5.35100-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["database","raven","ravendb","nosql","document"],"title":"RavenDB Database","version":"3.5.35100-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35100-unstable/ravendb.database.3.5.35100-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35101-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.04/ravendb.database.3.5.35101-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.04/ravendb.database.3.5.35101-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.14.04/ravendb.database.3.5.35101-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35101-Unstable, 3.5.35101-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35101-unstable/ravendb.database.3.5.35101-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","document","database","nosql","raven"],"title":"RavenDB Database","version":"3.5.35101-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35101-unstable/ravendb.database.3.5.35101-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35102-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35102-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35102-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35102-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35102-Unstable, 3.5.35102-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35102-unstable/ravendb.database.3.5.35102-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","database","raven","nosql","document"],"title":"RavenDB Database","version":"3.5.35102-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35102-unstable/ravendb.database.3.5.35102-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35103-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35103-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35103-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35103-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35103-Unstable, 3.5.35103-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35103-unstable/ravendb.database.3.5.35103-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","nosql","raven","database","document"],"title":"RavenDB Database","version":"3.5.35103-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35103-unstable/ravendb.database.3.5.35103-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35104-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35104-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35104-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35104-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35104-Unstable, 3.5.35104-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35104-unstable/ravendb.database.3.5.35104-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","document","database","raven","nosql"],"title":"RavenDB Database","version":"3.5.35104-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35104-unstable/ravendb.database.3.5.35104-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35105-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35105-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35105-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35105-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35105-Unstable, 3.5.35105-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35105-unstable/ravendb.database.3.5.35105-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["database","document","nosql","ravendb","raven"],"title":"RavenDB Database","version":"3.5.35105-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35105-unstable/ravendb.database.3.5.35105-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35106-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.13.24/ravendb.database.3.5.35106-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.13.24/ravendb.database.3.5.35106-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.13.24/ravendb.database.3.5.35106-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35106-Unstable, 3.5.35106-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35106-unstable/ravendb.database.3.5.35106-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["document","database","ravendb","nosql","raven"],"title":"RavenDB Database","version":"3.5.35106-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35106-unstable/ravendb.database.3.5.35106-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35107-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.06/ravendb.database.3.5.35107-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.06/ravendb.database.3.5.35107-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.06/ravendb.database.3.5.35107-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35107-Unstable, 3.5.35107-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35107-unstable/ravendb.database.3.5.35107-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["nosql","database","ravendb","raven","document"],"title":"RavenDB Database","version":"3.5.35107-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35107-unstable/ravendb.database.3.5.35107-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35108-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.59/ravendb.database.3.5.35108-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.59/ravendb.database.3.5.35108-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.59/ravendb.database.3.5.35108-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35108-Unstable, 3.5.35108-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35108-unstable/ravendb.database.3.5.35108-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["document","database","raven","nosql","ravendb"],"title":"RavenDB Database","version":"3.5.35108-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35108-unstable/ravendb.database.3.5.35108-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35109-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35109-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35109-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35109-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35109-Unstable, 3.5.35109-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35109-unstable/ravendb.database.3.5.35109-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["database","nosql","raven","ravendb","document"],"title":"RavenDB Database","version":"3.5.35109-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35109-unstable/ravendb.database.3.5.35109-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35110-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.59/ravendb.database.3.5.35110-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.59/ravendb.database.3.5.35110-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.59/ravendb.database.3.5.35110-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35110-Unstable, 3.5.35110-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35110-unstable/ravendb.database.3.5.35110-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["nosql","document","ravendb","database","raven"],"title":"RavenDB Database","version":"3.5.35110-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35110-unstable/ravendb.database.3.5.35110-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35111-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35111-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35111-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.12.45/ravendb.database.3.5.35111-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35111-Unstable, 3.5.35111-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35111-unstable/ravendb.database.3.5.35111-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","database","nosql","raven","document"],"title":"RavenDB Database","version":"3.5.35111-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35111-unstable/ravendb.database.3.5.35111-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35112-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.21/ravendb.database.3.5.35112-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.21/ravendb.database.3.5.35112-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.21/ravendb.database.3.5.35112-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35112-Unstable, 3.5.35112-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35112-unstable/ravendb.database.3.5.35112-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["document","nosql","database","ravendb","raven"],"title":"RavenDB Database","version":"3.5.35112-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35112-unstable/ravendb.database.3.5.35112-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35113-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.14/ravendb.database.3.5.35113-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.14/ravendb.database.3.5.35113-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.14/ravendb.database.3.5.35113-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35113-Unstable, 3.5.35113-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35113-unstable/ravendb.database.3.5.35113-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","nosql","raven","document","database"],"title":"RavenDB Database","version":"3.5.35113-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35113-unstable/ravendb.database.3.5.35113-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35114-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.14/ravendb.database.3.5.35114-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.14/ravendb.database.3.5.35114-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.14/ravendb.database.3.5.35114-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35114-Unstable, 3.5.35114-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35114-unstable/ravendb.database.3.5.35114-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","nosql","raven","document","database"],"title":"RavenDB Database","version":"3.5.35114-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35114-unstable/ravendb.database.3.5.35114-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35115-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.59/ravendb.database.3.5.35115-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.59/ravendb.database.3.5.35115-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.59/ravendb.database.3.5.35115-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35115-Unstable, 3.5.35115-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35115-unstable/ravendb.database.3.5.35115-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["raven","database","document","nosql","ravendb"],"title":"RavenDB Database","version":"3.5.35115-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35115-unstable/ravendb.database.3.5.35115-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35117-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.14/ravendb.database.3.5.35117-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.14/ravendb.database.3.5.35117-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.14/ravendb.database.3.5.35117-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35117-Unstable, 3.5.35117-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35117-unstable/ravendb.database.3.5.35117-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["database","nosql","ravendb","raven","document"],"title":"RavenDB Database","version":"3.5.35117-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35117-unstable/ravendb.database.3.5.35117-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35118-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.36/ravendb.database.3.5.35118-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.36/ravendb.database.3.5.35118-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.36/ravendb.database.3.5.35118-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35118-Unstable, 3.5.35118-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35118-unstable/ravendb.database.3.5.35118-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["document","ravendb","database","raven","nosql"],"title":"RavenDB Database","version":"3.5.35118-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35118-unstable/ravendb.database.3.5.35118-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35119-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35119-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35119-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35119-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35119-Unstable, 3.5.35119-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35119-unstable/ravendb.database.3.5.35119-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["nosql","database","raven","ravendb","document"],"title":"RavenDB Database","version":"3.5.35119-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35119-unstable/ravendb.database.3.5.35119-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35120-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35120-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35120-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35120-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35120-Unstable, 3.5.35120-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35120-unstable/ravendb.database.3.5.35120-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["raven","nosql","ravendb","document","database"],"title":"RavenDB Database","version":"3.5.35120-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35120-unstable/ravendb.database.3.5.35120-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35121-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.14/ravendb.database.3.5.35121-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.14/ravendb.database.3.5.35121-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.11.14/ravendb.database.3.5.35121-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35121-Unstable, 3.5.35121-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35121-unstable/ravendb.database.3.5.35121-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["raven","nosql","database","document","ravendb"],"title":"RavenDB Database","version":"3.5.35121-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35121-unstable/ravendb.database.3.5.35121-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35122-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.50/ravendb.database.3.5.35122-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.50/ravendb.database.3.5.35122-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.50/ravendb.database.3.5.35122-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35122-Unstable, 3.5.35122-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35122-unstable/ravendb.database.3.5.35122-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["database","ravendb","raven","document","nosql"],"title":"RavenDB Database","version":"3.5.35122-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35122-unstable/ravendb.database.3.5.35122-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35123-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.50/ravendb.database.3.5.35123-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.50/ravendb.database.3.5.35123-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.50/ravendb.database.3.5.35123-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35123-Unstable, 3.5.35123-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35123-unstable/ravendb.database.3.5.35123-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["database","nosql","raven","ravendb","document"],"title":"RavenDB Database","version":"3.5.35123-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35123-unstable/ravendb.database.3.5.35123-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35124-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35124-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35124-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35124-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35124-Unstable, 3.5.35124-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35124-unstable/ravendb.database.3.5.35124-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","nosql","document","raven","database"],"title":"RavenDB Database","version":"3.5.35124-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35124-unstable/ravendb.database.3.5.35124-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35125-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35125-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35125-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35125-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35125-Unstable, 3.5.35125-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35125-unstable/ravendb.database.3.5.35125-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["raven","nosql","database","document","ravendb"],"title":"RavenDB Database","version":"3.5.35125-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35125-unstable/ravendb.database.3.5.35125-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35126-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35126-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35126-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.10.29/ravendb.database.3.5.35126-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35126-Unstable, 3.5.35126-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35126-unstable/ravendb.database.3.5.35126-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","nosql","database","document","raven"],"title":"RavenDB Database","version":"3.5.35126-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35126-unstable/ravendb.database.3.5.35126-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35127-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35127-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35127-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35127-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35127-Unstable, 3.5.35127-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35127-unstable/ravendb.database.3.5.35127-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["document","raven","nosql","ravendb","database"],"title":"RavenDB Database","version":"3.5.35127-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35127-unstable/ravendb.database.3.5.35127-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35128-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35128-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35128-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35128-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35128-Unstable, 3.5.35128-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35128-unstable/ravendb.database.3.5.35128-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","nosql","raven","database","document"],"title":"RavenDB Database","version":"3.5.35128-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35128-unstable/ravendb.database.3.5.35128-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35129-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35129-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35129-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35129-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35129-Unstable, 3.5.35129-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35129-unstable/ravendb.database.3.5.35129-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["document","ravendb","nosql","raven","database"],"title":"RavenDB Database","version":"3.5.35129-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35129-unstable/ravendb.database.3.5.35129-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35130-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35130-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35130-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35130-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35130-Unstable, 3.5.35130-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35130-unstable/ravendb.database.3.5.35130-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["raven","document","database","nosql","ravendb"],"title":"RavenDB Database","version":"3.5.35130-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35130-unstable/ravendb.database.3.5.35130-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35132-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.08.23/ravendb.database.3.5.35132-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.08.23/ravendb.database.3.5.35132-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.08.23/ravendb.database.3.5.35132-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35132-Unstable, 3.5.35132-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35132-unstable/ravendb.database.3.5.35132-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["database","document","ravendb","nosql","raven"],"title":"RavenDB Database","version":"3.5.35132-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35132-unstable/ravendb.database.3.5.35132-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35133-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.08.23/ravendb.database.3.5.35133-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.08.23/ravendb.database.3.5.35133-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.08.23/ravendb.database.3.5.35133-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35133-Unstable, 3.5.35133-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35133-unstable/ravendb.database.3.5.35133-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","nosql","database","document","raven"],"title":"RavenDB Database","version":"3.5.35133-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35133-unstable/ravendb.database.3.5.35133-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35134-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35134-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35134-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35134-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35134-Unstable, 3.5.35134-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35134-unstable/ravendb.database.3.5.35134-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["document","ravendb","database","raven","nosql"],"title":"RavenDB Database","version":"3.5.35134-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35134-unstable/ravendb.database.3.5.35134-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35135-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35135-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35135-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.09.05/ravendb.database.3.5.35135-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35135-Unstable, 3.5.35135-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35135-unstable/ravendb.database.3.5.35135-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["database","document","raven","ravendb","nosql"],"title":"RavenDB Database","version":"3.5.35135-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35135-unstable/ravendb.database.3.5.35135-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35136-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.07.45/ravendb.database.3.5.35136-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.07.45/ravendb.database.3.5.35136-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.07.45/ravendb.database.3.5.35136-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35136-Unstable, 3.5.35136-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35136-unstable/ravendb.database.3.5.35136-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["ravendb","document","nosql","database","raven"],"title":"RavenDB Database","version":"3.5.35136-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35136-unstable/ravendb.database.3.5.35136-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"},{"@id":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/3.5.35142-unstable.json","@type":"Package","commitId":"11185fcc-dd1b-461f-b5c6-0c06068c0a6d","commitTimeStamp":"2018-11-07T08:01:25.5414875Z","catalogEntry":{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.07.38/ravendb.database.3.5.35142-unstable.json","@type":"PackageDetails","authors":"Hibernating Rhinos","dependencyGroups":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.07.38/ravendb.database.3.5.35142-unstable.json#dependencygroup","@type":"PackageDependencyGroup","dependencies":[{"@id":"https://api.nuget.org/v3/catalog0/data/2018.10.20.06.07.38/ravendb.database.3.5.35142-unstable.json#dependencygroup/ravendb.client","@type":"PackageDependency","id":"RavenDB.Client","range":"[3.5.35142-Unstable, 3.5.35142-Unstable]","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.client/index.json"}]}],"description":"Use this package if you want extend RavenDB. Don't use this package if you just want to work with existing RavenDB server, in order to so just use the client API which is in the RavenDB.Client package. RavenDB is a document database for the .NET platform, offering a flexible data model design to fit the needs of real world systems.\n\nNote: If you encounter issue to install this package, please consult the following link: https://groups.google.com/forum/#!topic/ravendb/4TeMq7_7Esc","iconUrl":"http://static.ravendb.net/logo-for-nuget.png","id":"RavenDB.Database","language":"en-US","licenseUrl":"http://www.ravendb.net/licensing","listed":false,"minClientVersion":"","packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35142-unstable/ravendb.database.3.5.35142-unstable.nupkg","projectUrl":"http://www.ravendb.net/","published":"1900-01-01T00:00:00+00:00","requireLicenseAcceptance":true,"summary":"This package allows you to extend RavenDB database.","tags":["raven","database","nosql","ravendb","document"],"title":"RavenDB Database","version":"3.5.35142-Unstable"},"packageContent":"https://api.nuget.org/v3-flatcontainer/ravendb.database/3.5.35142-unstable/ravendb.database.3.5.35142-unstable.nupkg","registration":"https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/index.json"}],"@context":{"@vocab":"http://schema.nuget.org/schema#","catalog":"http://schema.nuget.org/catalog#","xsd":"http://www.w3.org/2001/XMLSchema#","items":{"@id":"catalog:item","@container":"@set"},"commitTimeStamp":{"@id":"catalog:commitTimeStamp","@type":"xsd:dateTime"},"commitId":{"@id":"catalog:commitId"},"count":{"@id":"catalog:count"},"parent":{"@id":"catalog:parent","@type":"@id"},"tags":{"@container":"@set","@id":"tag"},"packageTargetFrameworks":{"@container":"@set","@id":"packageTargetFramework"},"dependencyGroups":{"@container":"@set","@id":"dependencyGroup"},"dependencies":{"@container":"@set","@id":"dependency"},"packageContent":{"@type":"@id"},"published":{"@type":"xsd:dateTime"},"registration":{"@type":"@id"}}} + + + https://api.nuget.org/v3/registration3-gz-semver2/ravendb.database/page/3.5.35078-unstable/3.5.35142-unstable.json + + \ No newline at end of file diff --git a/tests/NuGet.Protocol.Catalog.Tests/TestDataHttpMessageHandler.cs b/tests/NuGet.Protocol.Catalog.Tests/TestDataHttpMessageHandler.cs new file mode 100644 index 000000000..9e91ec825 --- /dev/null +++ b/tests/NuGet.Protocol.Catalog.Tests/TestDataHttpMessageHandler.cs @@ -0,0 +1,57 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Protocol.Catalog +{ + public class TestDataHttpMessageHandler : HttpMessageHandler + { + private static readonly Dictionary> UrlToGetContent = new Dictionary> + { + { TestData.CatalogIndexUrl, () => TestData.CatalogIndex }, + { TestData.CatalogPageUrl, () => TestData.CatalogPage }, + { TestData.PackageDeleteCatalogLeafUrl, () => TestData.PackageDeleteCatalogLeaf }, + { TestData.PackageDetailsCatalogLeafUrl, () => TestData.PackageDetailsCatalogLeaf }, + { TestData.RegistrationIndexInlinedItemsUrl, () => TestData.RegistrationIndexInlinedItems }, + { TestData.RegistrationIndexWithoutInlinedItemsUrl, () => TestData.RegistrationIndexWithoutInlinedItems }, + { TestData.RegistrationLeafUnlistedUrl, () => TestData.RegistrationLeafUnlisted }, + { TestData.RegistrationLeafListedUrl, () => TestData.RegistrationLeafListed }, + { TestData.RegistrationPageUrl, () => TestData.RegistrationPage }, + { TestData.CatalogLeafInvalidDependencyVersionRangeUrl, () => TestData.CatalogLeafInvalidDependencyVersionRange }, + { TestData.PackageDetailsLeafWithRequireLicenseAcceptanceUrl, () => TestData.PackageDetailsLeafWithRequireLicenseAcceptance }, + }; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return Task.FromResult(Send(request)); + } + + private HttpResponseMessage Send(HttpRequestMessage request) + { + Func getContent; + if (request.Method != HttpMethod.Get + || !UrlToGetContent.TryGetValue(request.RequestUri.AbsoluteUri, out getContent)) + { + return new HttpResponseMessage + { + RequestMessage = request, + StatusCode = HttpStatusCode.NotFound, + Content = new StringContent(string.Empty), + }; + } + + return new HttpResponseMessage + { + RequestMessage = request, + StatusCode = HttpStatusCode.OK, + Content = new StringContent(getContent()), + }; + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Analysis/DescriptionCustomAnalyzerFunctionalTests.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Analysis/DescriptionCustomAnalyzerFunctionalTests.cs new file mode 100644 index 000000000..d8a75a085 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Analysis/DescriptionCustomAnalyzerFunctionalTests.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using Xunit; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class DescriptionCustomAnalyzerFacts : AzureSearchIndexFunctionalTestBase + { + private const string AnalyzerName = "nuget_description_analyzer"; + + public DescriptionCustomAnalyzerFacts(CommonFixture fixture) + : base(fixture) + { + } + + [AnalysisTheory] + [MemberData(nameof(TokenizationData.LowercasesTokens), MemberType = typeof(TokenizationData))] + [MemberData(nameof(TokenizationData.TruncatesTokensAtLength300), MemberType = typeof(TokenizationData))] + [MemberData(nameof(TokenizationData.SplitsTokensOnSpecialCharactersAndLowercases), MemberType = typeof(TokenizationData))] + [MemberData(nameof(TokenizationData.LowercasesAndAddsTokensOnCasingAndNonAlphaNumeric), MemberType = typeof(TokenizationData))] + [MemberData(nameof(TokenizationData.AddsTokensOnNonAlphaNumericAndRemovesStopWords), MemberType = typeof(TokenizationData))] + public async Task ProducesExpectedTokens(string input, string[] expectedTokens) + { + var actualTokens = new HashSet(await AnalyzeAsync(AnalyzerName, input)); + + foreach (var expectedToken in expectedTokens) + { + Assert.Contains(expectedToken, actualTokens); + } + + Assert.Equal(expectedTokens.Length, actualTokens.Count); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Analysis/ExactMatchCustomAnalyzerFunctionalTests.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Analysis/ExactMatchCustomAnalyzerFunctionalTests.cs new file mode 100644 index 000000000..1c3f96da6 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Analysis/ExactMatchCustomAnalyzerFunctionalTests.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Xunit; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class ExactMatchCustomAnalyzerFacts : AzureSearchIndexFunctionalTestBase + { + private const string AnalyzerName = "nuget_exact_match_analyzer"; + + public ExactMatchCustomAnalyzerFacts(CommonFixture fixture) + : base(fixture) + { + } + + [AnalysisFact] + public async Task LowercasesInput() + { + var tokens = await AnalyzeAsync(AnalyzerName, "Hello world. FooBarBaz 𠈓"); + + var token = Assert.Single(tokens); + Assert.Equal("hello world. foobarbaz 𠈓", token); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Analysis/PackageIdCustomAnalyzer.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Analysis/PackageIdCustomAnalyzer.cs new file mode 100644 index 000000000..c98d1bc3a --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Analysis/PackageIdCustomAnalyzer.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using Xunit; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class PackageIdCustomAnalyzerFacts : AzureSearchIndexFunctionalTestBase + { + private const string AnalyzerName = "nuget_package_id_analyzer"; + + public PackageIdCustomAnalyzerFacts(CommonFixture fixture) + : base(fixture) + { + } + + [AnalysisTheory] + [MemberData(nameof(TokenizationData.LowercasesTokens), MemberType = typeof(TokenizationData))] + [MemberData(nameof(TokenizationData.TruncatesTokensAtLength300), MemberType = typeof(TokenizationData))] + [MemberData(nameof(TokenizationData.SplitsTokensOnSpecialCharactersAndLowercases), MemberType = typeof(TokenizationData))] + [MemberData(nameof(TokenizationData.LowercasesAndAddsTokensOnCasingAndNonAlphaNumeric), MemberType = typeof(TokenizationData))] + public async Task ProducesExpectedTokens(string input, string[] expectedTokens) + { + var actualTokens = new HashSet(await AnalyzeAsync(AnalyzerName, input)); + + foreach (var expectedToken in expectedTokens) + { + Assert.Contains(expectedToken, actualTokens); + } + + Assert.Equal(expectedTokens.Length, actualTokens.Count); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Analysis/TagsCustomAnalyzerFunctionalTests.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Analysis/TagsCustomAnalyzerFunctionalTests.cs new file mode 100644 index 000000000..3d28c6251 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Analysis/TagsCustomAnalyzerFunctionalTests.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Xunit; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class TagsCustomAnalyzerFunctionalTests : AzureSearchIndexFunctionalTestBase + { + private const string AnalyzerName = "nuget_tags_analyzer"; + + public TagsCustomAnalyzerFunctionalTests(CommonFixture fixture) + : base(fixture) + { + } + + [AnalysisTheory] + [MemberData(nameof(ProducesExpectedTokensData))] + public async Task ProducesExpectedTokens(string input, string[] expectedTokens) + { + var actualTokens = new HashSet(await AnalyzeAsync(AnalyzerName, input)); + + foreach (var expectedToken in expectedTokens) + { + Assert.Contains(expectedToken, actualTokens); + } + + Assert.Equal(expectedTokens.Length, actualTokens.Count); + } + + public static IEnumerable ProducesExpectedTokensData() + { + var tests = new List(); + + tests.AddRange(TokenizationData.LowercasesTokens); + tests.AddRange(TokenizationData.TrimsTokens); + tests.AddRange(TokenizationData.SplitsTokensAtLength300); + tests.AddRange(TokenizationData.DoesNotSplitTokensOnSpecialCharacters); + + // The gallery database stores tags up to length 4,000. + // Thus, the tags analyzer splits tokens into length 300 and removes duplicates, if any. + tests.Add(new object[] { new string('a', 600), new[] { new string('a', 300) } }); + + return tests; + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/BasicTests/AutocompleteProtocolTests.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/BasicTests/AutocompleteProtocolTests.cs new file mode 100644 index 000000000..8ec7d232a --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/BasicTests/AutocompleteProtocolTests.cs @@ -0,0 +1,115 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading.Tasks; +using BasicSearchTests.FunctionalTests.Core; +using BasicSearchTests.FunctionalTests.Core.TestSupport; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class AutocompleteProtocolTests : NuGetSearchFunctionalTestBase + { + public AutocompleteProtocolTests(CommonFixture fixture, ITestOutputHelper testOutputHelper) + : base(fixture, testOutputHelper) + { + } + + [Fact] + public async Task CanGetEmptyResult() + { + // Act + var result = await AutocompleteAsync(new AutocompleteBuilder { Query = Constants.NonExistentSearchString }); + + // Assert + Assert.Equal(0, result.TotalHits); + Assert.Empty(result.Data); + } + + [Fact] + public async Task ShouldGetResultsForEmptyString() + { + // Act + var result = await AutocompleteAsync(new AutocompleteBuilder { Query = "" }); + + // Assert + Assert.True(result.TotalHits.HasValue && result.TotalHits.Value > 0, "No results found, should find at least some results for empty string query."); + Assert.NotNull(result.Data); + } + + [Theory] + [InlineData("DOTnettOoL", "DotnetTool", Constants.TestPackageId_PackageType)] + [InlineData("depenDENCY", "Dependency", Constants.TestPackageId)] + public async Task TreatsPackageTypeAsCaseInsensitive(string packageTypeQuery, string expected, string id) + { + var searchBuilder = new AutocompleteBuilder + { + Query = id, + PackageType = packageTypeQuery, + }; + + var results = await AutocompleteAsync(searchBuilder); + + Assert.NotEmpty(results.Data); + Assert.Equal(id, results.Data[0]); + } + + [Fact] + public async Task IncludesPackagesWithMatchingPackageType() + { + var searchBuilder = new AutocompleteBuilder + { + Query = Constants.TestPackageId_PackageType, + PackageType = "DotNetTool", + }; + + var results = await AutocompleteAsync(searchBuilder); + + Assert.NotEmpty(results.Data); + Assert.Equal(Constants.TestPackageId_PackageType, results.Data[0]); + } + + [Fact] + public async Task PackagesWithoutPackageTypesAreAssumedToBeDependency() + { + var searchBuilder = new AutocompleteBuilder + { + Query = Constants.TestPackageId, + PackageType = "Dependency", + }; + + var results = await AutocompleteAsync(searchBuilder); + + Assert.NotEmpty(results.Data); + Assert.Equal(Constants.TestPackageId, results.Data[0]); + } + + [Fact] + public async Task ReturnsNothingWhenThePackageTypeDoesNotExist() + { + var searchBuilder = new AutocompleteBuilder + { + PackageType = Guid.NewGuid().ToString(), + }; + + var results = await AutocompleteAsync(searchBuilder); + + Assert.Empty(results.Data); + } + + [Fact] + public async Task ReturnsNothingWhenThePackageTypeIsInvalid() + { + var searchBuilder = new AutocompleteBuilder + { + PackageType = "cannot$be:a;package|type", + }; + + var results = await AutocompleteAsync(searchBuilder); + + Assert.Empty(results.Data); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/BasicTests/SearchAvailabilityTests.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/BasicTests/SearchAvailabilityTests.cs new file mode 100644 index 000000000..6a26e49b7 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/BasicTests/SearchAvailabilityTests.cs @@ -0,0 +1,26 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class SearchAvailabilityTests : NuGetSearchFunctionalTestBase + { + public SearchAvailabilityTests(CommonFixture fixture, ITestOutputHelper testOutputHelper) + : base(fixture, testOutputHelper) + { + } + + [Fact] + public async Task TheSearchEndpointReturnsResults() + { + var results = await SearchAsync(""); + + Assert.NotNull(results); + Assert.True(results.Count > 1); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/BasicTests/V2SearchProtocolTests.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/BasicTests/V2SearchProtocolTests.cs new file mode 100644 index 000000000..a8a64036a --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/BasicTests/V2SearchProtocolTests.cs @@ -0,0 +1,693 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using BasicSearchTests.FunctionalTests.Core; +using BasicSearchTests.FunctionalTests.Core.Models; +using BasicSearchTests.FunctionalTests.Core.TestSupport; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class V2SearchProtocolTests : NuGetSearchFunctionalTestBase + { + public V2SearchProtocolTests(CommonFixture fixture, ITestOutputHelper testOutputHelper) + : base(fixture, testOutputHelper) + { + } + + [Fact] + public async Task CanGetEmptyResult() + { + // Act + var result = await V2SearchAsync(new V2SearchBuilder { Query = Constants.NonExistentSearchString }); + + // Assert + Assert.Equal(0, result.TotalHits); + Assert.Empty(result.Data); + } + + [Fact] + public async Task ShouldGetResultsForEmptyString() + { + // Act + var result = await V2SearchAsync(new V2SearchBuilder { Query = "" }); + + // Assert + Assert.True(result.TotalHits.HasValue && result.TotalHits.Value > 0, "No results found, should find at least some results for empty string query."); + Assert.NotNull(result.Data); + } + + [Theory] + [MemberData(nameof(TakeResults))] + public async Task TakeReturnsExactResults(int take) + { + var results = await V2SearchAsync(new V2SearchBuilder { Query = "", Take = take }); + + Assert.NotNull(results); + Assert.True(results.TotalHits >= results.Data.Count); + Assert.True(results.Data.Count == take, $"The search result did not return the expected {take} results"); + } + + [Fact] + public async Task SkipDoesSkipThePackagesInResult() + { + var searchTerm = ""; + var skip = 5; + var resultsWithoutSkip = await V2SearchAsync(new V2SearchBuilder { Query = searchTerm, Take = 10 }); + var resultsWithSkip = await V2SearchAsync(new V2SearchBuilder { Query = searchTerm, Skip = skip, Take = 10 - skip }); + + Assert.NotNull(resultsWithoutSkip); + Assert.NotNull(resultsWithSkip); + Assert.True(resultsWithoutSkip.Data.Count > 1); + Assert.True(resultsWithSkip.Data.Count > 1); + Assert.True(resultsWithoutSkip.Data.Count == resultsWithSkip.Data.Count + skip); + var packageIDListThatShouldBeSkipped = resultsWithoutSkip.Data + .Select(x => x.PackageRegistration.Id) + .Take(skip); + var commonResults = resultsWithSkip.Data + .Select(x => x.PackageRegistration.Id) + .Where(x => packageIDListThatShouldBeSkipped.Contains(x, StringComparer.OrdinalIgnoreCase)); + Assert.True(commonResults.Count() == 0, $"Found results that should have been skipped"); + } + + [Fact] + public async Task CountOnlyReturnsCountOfSearchResults() + { + var results = await V2SearchAsync(new V2SearchBuilder { CountOnly = true }); + + Assert.NotNull(results); + Assert.True(results.TotalHits > 1); + Assert.Null(results.Data); + } + + /// + /// This is the query pattern used by gallery to handle "FindPackagesById()?id={id}" OData queries. + /// + [Fact] + public async Task ODataFindPackagesById() + { + var results = await V2SearchAsync(new V2SearchBuilder + { + Query = $"Id:\"{Constants.TestPackageId}\"", + Skip = 0, + Take = 100, + SortBy = "relevance", + IncludeSemVer2 = true, + Prerelease = true, + IgnoreFilter = true, + LuceneQuery = null, + }); + + Assert.NotNull(results); + Assert.True(results.TotalHits >= 1); + Assert.NotEmpty(results.Data); + foreach (var result in results.Data) + { + Assert.Equal(Constants.TestPackageId, result.PackageRegistration.Id); + } + } + + /// + /// This is the query pattern used by gallery to handle "Packages(Id='{id}',Version='{version}')" OData queries. + /// + [Fact] + public async Task ODataSpecificPackage() + { + var results = await V2SearchAsync(new V2SearchBuilder + { + Query = $"Id:\"{Constants.TestPackageId}\" AND Version:\"{Constants.TestPackageVersion}\"", + Skip = 0, + Take = 1, + SortBy = "relevance", + IncludeSemVer2 = true, + Prerelease = true, + IgnoreFilter = true, + LuceneQuery = null, + }); + + Assert.NotNull(results); + Assert.Equal(1, results.TotalHits); + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId, package.PackageRegistration.Id); + Assert.Equal(Constants.TestPackageVersion, package.NormalizedVersion); + } + + [Fact] + public async Task ResultsHonorPreReleaseField() + { + var searchTerm = "basetestpackage"; + + var resultsWithPrerelease = await V2SearchAsync(new V2SearchBuilder { Query = searchTerm, Prerelease = true }); ; + Assert.NotNull(resultsWithPrerelease); + Assert.True(resultsWithPrerelease.Data.Count > 1); + + var resultsWithoutPrerelease = await V2SearchAsync(new V2SearchBuilder { Query = searchTerm, Prerelease = false }); + Assert.NotNull(resultsWithoutPrerelease); + Assert.True(resultsWithoutPrerelease.Data.Count > 1); + + // The result count should be different for included prerelease results, + // else it means the search term responded with same results i.e. no results have prerelease versions + Assert.True(resultsWithPrerelease.TotalHits > resultsWithoutPrerelease.TotalHits, + $"The search term {searchTerm} does not seem to have any prerelease versions in the search index."); + + var hasPrereleaseVersions = resultsWithPrerelease + .Data + .Any(x => TestUtilities.IsPrerelease(x.Version)); + Assert.True(hasPrereleaseVersions, $"The search query did not return any results with the expected prerelease versions."); + + hasPrereleaseVersions = resultsWithoutPrerelease + .Data + .Any(x => TestUtilities.IsPrerelease(x.Version)); + Assert.False(hasPrereleaseVersions, $"The search query returned results with prerelease versions when queried for Prerelease = false"); + } + + [Theory] + [InlineData("packageid:" + Constants.TestPackageId_Unlisted)] + [InlineData("packageid:" + Constants.TestPackageId_Unlisted + " version:" + Constants.TestPackageVersion_Unlisted)] + public async Task HidesUnlistedPackagesByDefault(string query) + { + var searchBuilder = new V2SearchBuilder + { + Query = query, + }; + + var results = await V2SearchAsync(searchBuilder); + + Assert.Empty(results.Data); + } + + [Theory] + [InlineData("packageid:" + Constants.TestPackageId_Unlisted)] + [InlineData("packageid:" + Constants.TestPackageId_Unlisted + " version:" + Constants.TestPackageVersion_Unlisted)] + public async Task ShowsUnlistedPackagesWithIgnoreFilterTrue(string query) + { + var searchBuilder = new V2SearchBuilder + { + Query = query, + IgnoreFilter = true, + }; + + var results = await V2SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId_Unlisted, package.PackageRegistration.Id); + Assert.Equal(Constants.TestPackageVersion_Unlisted, package.Version); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ComparesVersionAsCaseInsensitive(bool ignoreFilter) + { + var searchBuilder = new V2SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_SearchFilters} version:{Constants.TestPackageVersion_SearchFilters_PrerelSemVer2.ToUpperInvariant()}", + Prerelease = true, + IncludeSemVer2 = true, + IgnoreFilter = ignoreFilter, + }; + + var results = await V2SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId_SearchFilters, package.PackageRegistration.Id); + Assert.Equal(Constants.TestPackageVersion_SearchFilters_PrerelSemVer2, package.Version); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ComparesIdAsCaseInsensitive(bool ignoreFilter) + { + var searchBuilder = new V2SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_SearchFilters.ToUpperInvariant()}", + Prerelease = false, + IncludeSemVer2 = false, + IgnoreFilter = ignoreFilter, + }; + + var results = await V2SearchAsync(searchBuilder); + + var package = results.Data.OrderBy(x => x.Version).FirstOrDefault(); + Assert.NotNull(package); + Assert.Equal(Constants.TestPackageId_SearchFilters, package.PackageRegistration.Id); + Assert.Equal(Constants.TestPackageVersion_SearchFilters_Default, package.Version); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task NormalizesVersion(bool ignoreFilter) + { + var searchBuilder = new V2SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_SearchFilters} version:1.04.0.0-delta.4+git", + Prerelease = true, + IncludeSemVer2 = true, + IgnoreFilter = ignoreFilter, + }; + + var results = await V2SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId_SearchFilters, package.PackageRegistration.Id); + Assert.Equal(Constants.TestPackageVersion_SearchFilters_PrerelSemVer2, package.Version); + } + + [Fact] + public async Task ByDefaultReturnsAnyPackageType() + { + var searchBuilder = new V2SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_PackageType}", + }; + + var results = await V2SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId_PackageType, package.PackageRegistration.Id); + Assert.Equal(Constants.TestPackageVersion_PackageType, package.Version); + } + + [Fact] + public async Task ExcludesPackagesWithMismatchingPackageType() + { + var searchBuilder = new V2SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_PackageType}", + PackageType = "Dependency", + }; + + var results = await V2SearchAsync(searchBuilder); + + Assert.Empty(results.Data); + } + + [Theory] + [InlineData("DOTnettOoL", "DotnetTool", Constants.TestPackageId_PackageType)] + [InlineData("depenDENCY", "Dependency", Constants.TestPackageId)] + public async Task TreatsPackageTypeAsCaseInsensitive(string packageTypeQuery, string expected, string id) + { + var searchBuilder = new V2SearchBuilder + { + Query = $"packageid:{id}", + PackageType = packageTypeQuery, + }; + + var results = await V2SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(id, package.PackageRegistration.Id); + } + + [Fact] + public async Task IncludesPackagesWithMatchingPackageType() + { + var searchBuilder = new V2SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_PackageType}", + PackageType = "DotNetTool", + }; + + var results = await V2SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId_PackageType, package.PackageRegistration.Id); + Assert.Equal(Constants.TestPackageVersion_PackageType, package.Version); + } + + [Fact] + public async Task PackagesWithoutPackageTypesAreAssumedToBeDependency() + { + var searchBuilder = new V2SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId}", + PackageType = "Dependency", + }; + + var results = await V2SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId, package.PackageRegistration.Id); + Assert.Equal(Constants.TestPackageVersion, package.Version); + } + + [Fact] + public async Task ReturnsNothingWhenThePackageTypeDoesNotExist() + { + var searchBuilder = new V2SearchBuilder + { + PackageType = Guid.NewGuid().ToString(), + }; + + var results = await V2SearchAsync(searchBuilder); + + Assert.Empty(results.Data); + } + + [Fact] + public async Task ReturnsNothingWhenThePackageTypeIsInvalid() + { + var searchBuilder = new V2SearchBuilder + { + PackageType = "cannot$be:a;package|type", + }; + + var results = await V2SearchAsync(searchBuilder); + + Assert.Empty(results.Data); + } + + [Fact] + public async Task ReturnsErrorWhenUsingPackageTypeFilterOnHijack() + { + var searchBuilder = new V2SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_PackageType}", + PackageType = "DotNetTool", + IgnoreFilter = true, + }; + + var ex = await Assert.ThrowsAsync(async () => + { + var results = await V2SearchAsync(searchBuilder); + }); + + Assert.Equal("Response status code does not indicate success: 400 (Bad Request).", ex.Message); + } + + [Fact] + public async Task ReturnsErrorWhenSortingByDownloadsAscOnHijack() + { + var searchBuilder = new V2SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_PackageType}", + IgnoreFilter = true, + SortBy = "totalDownloads-asc" + }; + + var ex = await Assert.ThrowsAsync(async () => + { + var results = await V2SearchAsync(searchBuilder); + }); + + Assert.Equal("Response status code does not indicate success: 400 (Bad Request).", ex.Message); + } + + [Fact] + public async Task ReturnsErrorWhenSortingByDownloadsDescOnHijack() + { + var searchBuilder = new V2SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_PackageType}", + IgnoreFilter = true, + SortBy = "totalDownloads-desc" + }; + + var ex = await Assert.ThrowsAsync(async () => + { + var results = await V2SearchAsync(searchBuilder); + }); + + Assert.Equal("Response status code does not indicate success: 400 (Bad Request).", ex.Message); + } + + [Theory] + [InlineData(false, false, Constants.TestPackageVersion_SearchFilters_Default)] + [InlineData(true, false, Constants.TestPackageVersion_SearchFilters_Prerel)] + [InlineData(false, true, Constants.TestPackageVersion_SearchFilters_SemVer2)] + [InlineData(true, true, Constants.TestPackageVersion_SearchFilters_PrerelSemVer2)] + public async Task LatestVersionChangesWithRespectToSearchFilters(bool prerelease, bool includeSemVer2, string version) + { + var searchBuilder = new V2SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_SearchFilters}", + Prerelease = prerelease, + IncludeSemVer2 = includeSemVer2, + }; + + var results = await V2SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId_SearchFilters, package.PackageRegistration.Id); + Assert.Equal(version, package.Version); + } + + public static IEnumerable IgnoreFilterTrueData => new[] + { + new + { + Prerelease = false, + IncludeSemVer2 = false, + ExpectedVersions = new[] // prerelease is always returned with ignoreFilter=true + { + Constants.TestPackageVersion_SearchFilters_Default, + Constants.TestPackageVersion_SearchFilters_Prerel, + }, + }, + new + { + Prerelease = true, + IncludeSemVer2 = false, + ExpectedVersions = new[] + { + Constants.TestPackageVersion_SearchFilters_Default, + Constants.TestPackageVersion_SearchFilters_Prerel, + }, + }, + new + { + Prerelease = false, + IncludeSemVer2 = true, + ExpectedVersions = new[] // prerelease is always returned with ignoreFilter=true + { + Constants.TestPackageVersion_SearchFilters_Default, + Constants.TestPackageVersion_SearchFilters_Prerel, + Constants.TestPackageVersion_SearchFilters_SemVer2, + Constants.TestPackageVersion_SearchFilters_PrerelSemVer2, + }, + }, + new + { + Prerelease = true, + IncludeSemVer2 = true, + ExpectedVersions = new[] + { + Constants.TestPackageVersion_SearchFilters_Default, + Constants.TestPackageVersion_SearchFilters_Prerel, + Constants.TestPackageVersion_SearchFilters_SemVer2, + Constants.TestPackageVersion_SearchFilters_PrerelSemVer2, + }, + }, + }.Select(x => new object[] { x.Prerelease, x.IncludeSemVer2, x.ExpectedVersions }); + + [Theory] + [MemberData(nameof(IgnoreFilterTrueData))] + public async Task IgnoreFilterTrueAlwaysIncludesPrerelease(bool prerelease, bool includeSemVer2, string[] expectedVersions) + { + var searchBuilder = new V2SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_SearchFilters}", + Prerelease = prerelease, + IncludeSemVer2 = includeSemVer2, + IgnoreFilter = true, + }; + + var results = await V2SearchAsync(searchBuilder); + + Assert.Equal( + expectedVersions, + results.Data.Select(x => x.Version).OrderBy(x => x).ToArray()); + } + + [Theory] + [InlineData(false, false, false)] + [InlineData(true, false, false)] + [InlineData(false, true, true)] + [InlineData(true, true, true)] + public async Task IgnoreFilterTrueWithSpecificIdVersionAlwaysIncludesPrerelease(bool prerelease, bool includeSemVer2, bool returned) + { + var searchBuilder = new V2SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_SearchFilters} version:{Constants.TestPackageVersion_SearchFilters_PrerelSemVer2}", + Prerelease = prerelease, + IncludeSemVer2 = includeSemVer2, + IgnoreFilter = true, + }; + + var results = await V2SearchAsync(searchBuilder); + + if (returned) + { + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId_SearchFilters, package.PackageRegistration.Id); + Assert.Equal(Constants.TestPackageVersion_SearchFilters_PrerelSemVer2, package.Version); + } + else + { + Assert.Empty(results.Data); + } + } + + [Fact] + public async Task SemVer2IsHiddenByDefault() + { + var searchTerm = "packageId:" + Constants.TestPackageId_SemVer2; + var results = await V2SearchAsync(new V2SearchBuilder { Query = searchTerm }); + + Assert.NotNull(results); + Assert.Empty(results.Data); + } + + [Fact] + public async Task SemVerLevel2AllowsSemVer2Packages() + { + var searchTerm = "packageId:" + Constants.TestPackageId_SemVer2; + var results = await V2SearchAsync(new V2SearchBuilder { Query = searchTerm, IncludeSemVer2 = true }); + + Assert.NotNull(results); + Assert.True(results.Data.Count >= 1); + var atleastOneResultWithSemVer2 = results + .Data + .Any(x => TestUtilities.IsSemVer2(x.Version)); + + Assert.True(atleastOneResultWithSemVer2, $"The search query did not return with any semver2 results"); + } + + [Theory] + [MemberData(nameof(GetSortByData))] + public async Task ResultsAreOrderedBySpecifiedParameter(string orderBy, Func GetPropertyValue, bool reverse = false) + { + var results = await V2SearchAsync(new V2SearchBuilder { Query = "", SortBy = orderBy }); + + Assert.NotNull(results); + Assert.True(results.Data.Count > 1); + + int count = results.Data.Count < 10 ? results.Data.Count : 10; + var topResults = results + .Data + .Select(GetPropertyValue) + .Take(count) + .ToList(); + + // Descending comparers + Func comparer = (x1, x2) => { + switch (x1) + { + case DateTime _: + return DateTime.Compare((DateTime)x1, (DateTime)x2) >= 0; + case string _: + return string.Compare(x1.ToString(), x2.ToString(), StringComparison.OrdinalIgnoreCase) >= 0; + case long _: + return (long)x1 >= (long) x2; + default: + throw new NotSupportedException($"Type {x1.GetType()} is not supported."); + } + }; + + for (int i = 1; i < topResults.Count(); i++) + { + if (reverse) + { + // flip for ascending comparison + Assert.True(comparer(topResults[i], topResults[i-1]), $"Results are not ordered in the ascending order for {orderBy} field query"); + } else + { + Assert.True(comparer(topResults[i - 1], topResults[i]), $"Results are not ordered in the descending order for {orderBy} field query"); + } + } + } + + [Fact] + public async Task ResultsMatchIdQueryFieldSearch() + { + var results = await V2SearchAsync(new V2SearchBuilder { Query = "packageId:" + Constants.TestPackageId }); + + Assert.NotNull(results); + Assert.Equal(1, results.Data.Count); + Assert.Equal(Constants.TestPackageId, results.Data.First().PackageRegistration.Id, StringComparer.OrdinalIgnoreCase); + } + + [Fact] + public async Task ResultsMatchAuthorQueryFieldSearch() + { + var results = await V2SearchAsync(new V2SearchBuilder { Query = "author:" + Constants.TestPackageAuthor }); + + Assert.NotNull(results); + + var authorResults = results + .Data + .Select(x => x.Authors); + + var resultsWithoutTestAuthor = authorResults + .Any(x => !x.ToLower().Contains(Constants.TestPackageAuthor.ToLower())); + + Assert.False(resultsWithoutTestAuthor, $"The query returned search results without author {Constants.TestPackageAuthor}"); + } + + [Fact] + public async Task ResultsMatchOwnersQueryFieldSearch() + { + var results = await V2SearchAsync(new V2SearchBuilder { Query = "owners:" + Constants.TestPackageOwner }); + + Assert.NotNull(results); + + var ownerResults = results + .Data + .Select(x => x.PackageRegistration.Owners); + + var resultsWithoutTestOwner = ownerResults + .Any(x => !x.Contains(Constants.TestPackageOwner, StringComparer.OrdinalIgnoreCase)); + + Assert.False(resultsWithoutTestOwner, $"The query returned search results without owner {Constants.TestPackageOwner}"); + } + + [Fact] + public async Task ResultsMatchTagsQueryFieldSearch() + { + var results = await V2SearchAsync(new V2SearchBuilder { Query = "tags:" + Constants.TestPackageTag }); + + Assert.NotNull(results); + + var tagResults = results + .Data + .Select(x => x.Tags); + + var resultsWithoutTestTags = tagResults + .Any(x => !x.ToLower().Contains(Constants.TestPackageTag.ToLower())); + + Assert.False(resultsWithoutTestTags, $"The query returned search results without tags {Constants.TestPackageTag}"); + } + + + public static IEnumerable TakeResults + { + get + { + return Enumerable.Range(1, 10).Select(i => new object[] { i }); + } + } + + public static IEnumerable GetSortByData + { + get + { + yield return new object[] { "lastEdited", (Func)((V2SearchResultEntry data) => { return data.LastEdited; }) }; + yield return new object[] { "published", (Func)((V2SearchResultEntry data) => { return data.Published; }) }; + yield return new object[] { "title-asc", (Func)((V2SearchResultEntry data) => { return data.Title; }), true }; + yield return new object[] { "title-desc", (Func)((V2SearchResultEntry data) => { return data.Title; }) }; + yield return new object[] { "created-asc", (Func)((V2SearchResultEntry data) => { return data.Created; }), true }; + yield return new object[] { "created-desc", (Func)((V2SearchResultEntry data) => { return data.Created; }) }; + yield return new object[] { "totalDownloads-asc", (Func)((V2SearchResultEntry data) => { return data.PackageRegistration.DownloadCount; }), true }; + yield return new object[] { "totalDownloads-desc", (Func)((V2SearchResultEntry data) => { return data.PackageRegistration.DownloadCount; }) }; + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/BasicTests/V3SearchProtocolTests.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/BasicTests/V3SearchProtocolTests.cs new file mode 100644 index 000000000..e207f6a3a --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/BasicTests/V3SearchProtocolTests.cs @@ -0,0 +1,432 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using BasicSearchTests.FunctionalTests.Core; +using BasicSearchTests.FunctionalTests.Core.TestSupport; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class V3SearchProtocolTests : NuGetSearchFunctionalTestBase + { + public V3SearchProtocolTests(CommonFixture fixture, ITestOutputHelper testOutputHelper) + : base(fixture, testOutputHelper) + { + } + + [Fact] + public async Task CanGetEmptyResult() + { + // Act + var result = await V3SearchAsync(new V3SearchBuilder { Query = Constants.NonExistentSearchString }); + + // Assert + Assert.Equal(0, result.TotalHits); + Assert.Empty(result.Data); + } + + [Fact] + public async Task ShouldGetResultsForEmptyString() + { + // Act + var result = await V3SearchAsync(new V3SearchBuilder { Query = "" }); + + // Assert + Assert.True(result.TotalHits.HasValue && result.TotalHits.Value > 0, "No results found, should find at least some results for empty string query."); + Assert.NotNull(result.Data); + } + + [Fact] + public async Task EnsureTestPackageIsValid() + { + // Act + var result = await V3SearchAsync(new V3SearchBuilder() { Query = Constants.TestPackageId }); + + // Assert + Assert.True(result.TotalHits.HasValue); + Assert.True(result.TotalHits.Value > 0, $"Could not find test package {Constants.TestPackageId}"); + Assert.True(result.Data.Count > 0, $"Could not find test package {Constants.TestPackageId}"); + + Assert.NotNull(result.AtContext); + Assert.True(result.AtContext.AtVocab == "http://schema.nuget.org/schema#"); + Assert.False(string.IsNullOrEmpty(result.AtContext.AtBase)); + + // Assert that the package result whose Id is "BaseTestPackage" with Version "1.0.0" + // matches exactly what is expected. + var package = result.Data + .Where(p => p.Id == Constants.TestPackageId) + .Where(p => p.Version == Constants.TestPackageVersion) + .FirstOrDefault(); + + Assert.NotNull(package); + Assert.False(string.IsNullOrEmpty(package.AtId)); + Assert.True(package.AtType == "Package"); + Assert.False(string.IsNullOrEmpty(package.Registration)); + Assert.True(package.Description == Constants.TestPackageDescription); + Assert.True(package.Summary == Constants.TestPackageSummary); + Assert.True(package.Title == Constants.TestPackageTitle); + Assert.True(package.Tags.Count() == 2); + Assert.True(package.Tags[0] == "Tag1"); + Assert.True(package.Tags[1] == "Tag2"); + Assert.True(package.Authors.Count() == 1); + Assert.True(package.Authors[0] == Constants.TestPackageAuthor); + Assert.True(package.TotalDownloads != default(long)); + Assert.True(package.Versions.Count() == 1); + Assert.True(package.Versions[0].Version == "1.0.0"); + Assert.True(package.Versions[0].Downloads != default(long)); + Assert.False(string.IsNullOrEmpty(package.Versions[0].AtId)); + } + + [Theory] + [MemberData(nameof(TakeResults))] + public async Task TakeReturnsExactResults(int take) + { + var results = await V3SearchAsync(new V3SearchBuilder { Query = "", Take = take }); + + Assert.NotNull(results); + Assert.True(results.TotalHits >= results.Data.Count); + Assert.True(results.Data.Count == take, $"The search result did not return the expected {take} results"); + } + + [Fact] + public async Task SkipDoesSkipThePackagesInResult() + { + var searchTerm = ""; + var skip = 5; + var resultsWithoutSkip = await V3SearchAsync(new V3SearchBuilder { Query = searchTerm, Take = 10 }); + var resultsWithSkip = await V3SearchAsync(new V3SearchBuilder { Query = searchTerm, Skip = skip, Take = 10 - skip }); + + Assert.NotNull(resultsWithoutSkip); + Assert.NotNull(resultsWithSkip); + Assert.True(resultsWithoutSkip.Data.Count > 1); + Assert.True(resultsWithSkip.Data.Count > 1); + Assert.True(resultsWithoutSkip.Data.Count == resultsWithSkip.Data.Count + skip); + var packageIDListThatShouldBeSkipped = resultsWithoutSkip.Data + .Select(x => x.Id) + .Take(skip); + var commonResults = resultsWithSkip.Data + .Select(x => x.Id) + .Where(x => packageIDListThatShouldBeSkipped.Contains(x, StringComparer.OrdinalIgnoreCase)); + Assert.True(commonResults.Count() == 0, $"Found results that should have been skipped"); + } + + [Fact] + public async Task ResultsHonorPreReleaseField() + { + var searchTerm = "basetestpackage"; + + var resultsWithPrerelease = await V3SearchAsync(new V3SearchBuilder { Query = searchTerm, Prerelease = true }); ; + Assert.NotNull(resultsWithPrerelease); + Assert.True(resultsWithPrerelease.Data.Count > 1); + + var resultsWithoutPrerelease = await V3SearchAsync(new V3SearchBuilder { Query = searchTerm, Prerelease = false }); + Assert.NotNull(resultsWithoutPrerelease); + Assert.True(resultsWithoutPrerelease.Data.Count > 1); + + // The result count should be different for included prerelease results, + // else it means the search term responded with same results i.e. no results have prerelease versions + Assert.True(resultsWithPrerelease.TotalHits > resultsWithoutPrerelease.TotalHits, + $"The search term {searchTerm} does not seem to have any prerelease versions in the search index."); + + var hasPrereleaseVersions = resultsWithPrerelease + .Data + .SelectMany(x => x.Versions) + .Any(x => TestUtilities.IsPrerelease(x.Version)); + Assert.True(hasPrereleaseVersions, $"The search query did not return any results with the expected prerelease versions."); + + hasPrereleaseVersions = resultsWithoutPrerelease + .Data + .SelectMany(x => x.Versions) + .Any(x => TestUtilities.IsPrerelease(x.Version)); + Assert.False(hasPrereleaseVersions, $"The search query returned results with prerelease versions when queried for Prerelease = false"); + } + + [Theory] + [InlineData("packageid:" + Constants.TestPackageId_Unlisted)] + [InlineData("packageid:" + Constants.TestPackageId_Unlisted + " version:" + Constants.TestPackageVersion_Unlisted)] + public async Task HidesUnlistedPackagesByDefault(string query) + { + var searchBuilder = new V3SearchBuilder + { + Query = query, + }; + + var results = await V3SearchAsync(searchBuilder); + + Assert.Empty(results.Data); + } + + [Fact] + public async Task ComparesVersionAsCaseInsensitive() + { + var searchBuilder = new V3SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_SearchFilters} version:{Constants.TestPackageVersion_SearchFilters_PrerelSemVer2.ToUpperInvariant()}", + Prerelease = true, + IncludeSemVer2 = true + }; + + var results = await V3SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId_SearchFilters, package.Id); + Assert.Equal(Constants.TestPackageVersion_SearchFilters_PrerelSemVer2, package.Version); + } + + [Fact] + public async Task ComparesIdAsCaseInsensitive() + { + var searchBuilder = new V3SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_SearchFilters.ToUpperInvariant()}", + Prerelease = false, + IncludeSemVer2 = false, + }; + + var results = await V3SearchAsync(searchBuilder); + + var package = results.Data.OrderBy(x => x.Version).FirstOrDefault(); + Assert.NotNull(package); + Assert.Equal(Constants.TestPackageId_SearchFilters, package.Id); + Assert.Equal(Constants.TestPackageVersion_SearchFilters_Default, package.Version); + } + + [Fact] + public async Task NormalizesVersion() + { + var searchBuilder = new V3SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_SearchFilters} version:1.04.0.0-delta.4+git", + Prerelease = true, + IncludeSemVer2 = true + }; + + var results = await V3SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId_SearchFilters, package.Id); + Assert.Equal(Constants.TestPackageVersion_SearchFilters_PrerelSemVer2, package.Version); + } + + [Fact] + public async Task ByDefaultReturnsAnyPackageType() + { + var searchBuilder = new V3SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_PackageType}", + }; + + var results = await V3SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId_PackageType, package.Id); + Assert.Equal(Constants.TestPackageVersion_PackageType, package.Version); + Assert.Equal("DotnetTool", Assert.Single(package.PackageTypes).Name); + } + + [Fact] + public async Task ExcludesPackagesWithMismatchingPackageType() + { + var searchBuilder = new V3SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_PackageType}", + PackageType = "Dependency", + }; + + var results = await V3SearchAsync(searchBuilder); + + Assert.Empty(results.Data); + } + + [Theory] + [InlineData("DOTnettOoL", "DotnetTool", Constants.TestPackageId_PackageType)] + [InlineData("depenDENCY", "Dependency", Constants.TestPackageId)] + public async Task TreatsPackageTypeAsCaseInsensitive(string packageTypeQuery, string expected, string id) + { + var searchBuilder = new V3SearchBuilder + { + Query = $"packageid:{id}", + PackageType = packageTypeQuery, + }; + + var results = await V3SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(id, package.Id); + Assert.Equal(expected, Assert.Single(package.PackageTypes).Name); + } + + [Fact] + public async Task IncludesPackagesWithMatchingPackageType() + { + var searchBuilder = new V3SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_PackageType}", + PackageType = "DotNetTool", + }; + + var results = await V3SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId_PackageType, package.Id); + Assert.Equal(Constants.TestPackageVersion_PackageType, package.Version); + Assert.Equal("DotnetTool", Assert.Single(package.PackageTypes).Name); + } + + [Fact] + public async Task PackagesWithoutPackageTypesAreAssumedToBeDependency() + { + var searchBuilder = new V3SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId}", + PackageType = "Dependency", + }; + + var results = await V3SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId, package.Id); + Assert.Equal(Constants.TestPackageVersion, package.Version); + Assert.Equal("Dependency", Assert.Single(package.PackageTypes).Name); + } + + [Fact] + public async Task ReturnsNothingWhenThePackageTypeDoesNotExist() + { + var searchBuilder = new V3SearchBuilder + { + PackageType = Guid.NewGuid().ToString(), + }; + + var results = await V3SearchAsync(searchBuilder); + + Assert.Empty(results.Data); + } + + [Fact] + public async Task ReturnsNothingWhenThePackageTypeIsInvalid() + { + var searchBuilder = new V3SearchBuilder + { + PackageType = "cannot$be:a;package|type", + }; + + var results = await V3SearchAsync(searchBuilder); + + Assert.Empty(results.Data); + } + + [Theory] + [InlineData(false, false, Constants.TestPackageVersion_SearchFilters_Default)] + [InlineData(true, false, Constants.TestPackageVersion_SearchFilters_Prerel)] + [InlineData(false, true, Constants.TestPackageVersion_SearchFilters_SemVer2)] + [InlineData(true, true, Constants.TestPackageVersion_SearchFilters_PrerelSemVer2)] + public async Task LatestVersionChangesWithRespectToSearchFilters(bool prerelease, bool includeSemVer2, string version) + { + var searchBuilder = new V3SearchBuilder + { + Query = $"packageid:{Constants.TestPackageId_SearchFilters}", + Prerelease = prerelease, + IncludeSemVer2 = includeSemVer2, + }; + + var results = await V3SearchAsync(searchBuilder); + + var package = Assert.Single(results.Data); + Assert.Equal(Constants.TestPackageId_SearchFilters, package.Id); + Assert.Equal(version, package.Version); + } + + [Fact] + public async Task SemVer2IsHiddenByDefault() + { + var searchTerm = "packageId:" + Constants.TestPackageId_SemVer2; + var results = await V3SearchAsync(new V3SearchBuilder { Query = searchTerm }); + + Assert.NotNull(results); + Assert.Empty(results.Data); + } + + [Fact] + public async Task SemVerLevel2AllowsSemVer2Packages() + { + var searchTerm = "packageId:" + Constants.TestPackageId_SemVer2; + var results = await V3SearchAsync(new V3SearchBuilder { Query = searchTerm, IncludeSemVer2 = true }); + + Assert.NotNull(results); + Assert.True(results.Data.Count >= 1); + var atleastOneResultWithSemVer2 = results + .Data + .Any(x => TestUtilities.IsSemVer2(x.Version)); + + Assert.True(atleastOneResultWithSemVer2, $"The search query did not return with any semver2 results"); + } + + [Fact] + public async Task ResultsMatchIdQueryFieldSearch() + { + var results = await V3SearchAsync(new V3SearchBuilder { Query = "packageId:" + Constants.TestPackageId }); + + Assert.NotNull(results); + Assert.Equal(1, results.Data.Count); + Assert.Equal(Constants.TestPackageId, results.Data.First().Id, StringComparer.OrdinalIgnoreCase); + } + + [Fact] + public async Task ResultsMatchAuthorQueryFieldSearch() + { + var results = await V3SearchAsync(new V3SearchBuilder { Query = "author:" + Constants.TestPackageAuthor }); + + Assert.NotNull(results); + + var authorResults = results + .Data + .Select(x => x.Authors); + + var resultsWithoutTestAuthor = authorResults + .Any(x => !x.Contains(Constants.TestPackageAuthor, StringComparer.OrdinalIgnoreCase)); + + Assert.False(resultsWithoutTestAuthor, $"The query returned search results without author {Constants.TestPackageAuthor}"); + } + + [Fact] + public async Task ResultsMatchOwnersQueryFieldSearch() + { + var results = await V3SearchAsync(new V3SearchBuilder { Query = "packageId:" + Constants.TestPackageId + " " + "owners:" + Constants.TestPackageOwner }); + + Assert.NotNull(results); + Assert.NotEmpty(results.Data); + Assert.Equal(Constants.TestPackageId, results.Data[0].Id); + } + + [Fact] + public async Task ResultsMatchTagsQueryFieldSearch() + { + var results = await V3SearchAsync(new V3SearchBuilder { Query = "tags:" + Constants.TestPackageTag }); + + Assert.NotNull(results); + + var tagResults = results + .Data + .Select(x => x.Tags); + + var resultsWithoutTestTags = tagResults + .Any(x => !x.Contains(Constants.TestPackageTag, StringComparer.OrdinalIgnoreCase)); + + Assert.False(resultsWithoutTestTags, $"The query returned search results without tags {Constants.TestPackageTag}"); + } + + public static IEnumerable TakeResults + { + get + { + return Enumerable.Range(1, 10).Select(i => new object[] { i }); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/NuGet.Services.AzureSearch.FunctionalTests.csproj b/tests/NuGet.Services.AzureSearch.FunctionalTests/NuGet.Services.AzureSearch.FunctionalTests.csproj new file mode 100644 index 000000000..133da99d2 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/NuGet.Services.AzureSearch.FunctionalTests.csproj @@ -0,0 +1,108 @@ + + + + + Debug + AnyCPU + {EAD54C54-E29E-43D5-AE7F-1C194B4EE948} + Library + Properties + NuGet.Services.AzureSearch.FunctionalTests + NuGet.Services.AzureSearch.FunctionalTests + v4.7.2 + 512 + + + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2.2.0 + + + 2.75.0 + + + 5.0.0-preview1.5707 + + + 2.4.1 + + + 2.4.1 + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + {eea7b6c1-0358-4e67-9d2a-e30b8ff9ff3d} + BasicSearchTests.FunctionalTests.Core + + + + + + + + + + + \ No newline at end of file diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Properties/AssemblyInfo.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..e2be6b96f --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("AzureSearchTests.FunctionalTests")] +[assembly: ComVisible(false)] +[assembly: Guid("ead54c54-e29e-43d5-ae7f-1c194b4ee948")] \ No newline at end of file diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Relevancy/AutocompleteRelevancyFunctionalTests.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Relevancy/AutocompleteRelevancyFunctionalTests.cs new file mode 100644 index 000000000..a0a9dd7d8 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Relevancy/AutocompleteRelevancyFunctionalTests.cs @@ -0,0 +1,74 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class AutocompleteRelevancyFunctionalTests : NuGetSearchFunctionalTestBase + { + public AutocompleteRelevancyFunctionalTests(CommonFixture fixture, ITestOutputHelper testOutputHelper) + : base(fixture, testOutputHelper) + { + } + + [RelevancyTheory] + [MemberData(nameof(EnsureFirstResultsData))] + public async Task EnsureFirstResults(string searchTerm, string[] expectedFirstResults) + { + var results = await AutocompleteAsync(searchTerm, take: 10); + + Assert.True(results.Count > expectedFirstResults.Length); + + for (var i = 0; i < expectedFirstResults.Length; i++) + { + Assert.True(expectedFirstResults[i] == results[i], $"Expected result '{expectedFirstResults[i]}' at index #{i} for query '{searchTerm}'"); + } + } + + [RelevancyTheory] + [MemberData(nameof(EnsureTopResultsData))] + public async Task EnsureTopResults(string searchTerm, string[] expectedTopResults) + { + var results = await AutocompleteAsync(searchTerm, take: 10); + + Assert.True(results.Count > expectedTopResults.Length); + + foreach (var expectedTopResult in expectedTopResults) + { + Assert.True(results.Contains(expectedTopResult), $"Expected result '{expectedTopResult}' for query '{searchTerm}'"); + } + } + + public static IEnumerable EnsureFirstResultsData() + { + yield return new object[] { "aws", new[] { "awssdk.core" } }; + yield return new object[] { "log4", new[] { "log4net" } }; + yield return new object[] { "dap", new[] { "dapper" } }; + yield return new object[] { "json", new[] { "json" } }; + yield return new object[] { "jso", new[] { "newtonsoft.json" } }; + yield return new object[] { "entityframeworkcore.relational", new[] { "microsoft.entityframeworkcore.relational" } }; + yield return new object[] { "core.mvc.razor", new[] { "microsoft.aspnetcore.mvc.razor" } }; + yield return new object[] { "microsoft.aspnet.mvc", new[] { "microsoft.aspnet.mvc" } }; + } + + public static IEnumerable EnsureTopResultsData() + { + yield return new object[] { "extensions.log", new[] { "microsoft.extensions.logging" } }; + yield return new object[] { "extensions.logging", new[] { "microsoft.extensions.logging" } }; + yield return new object[] { "microsoft.extensio", new[] { "microsoft.extensions.logging.abstractions" } }; + yield return new object[] { "depen", new[] { "microsoft.extensions.dependencyinjection" } }; + yield return new object[] { "ent", new[] { "entityframework", "microsoft.entityframeworkcore" } }; + yield return new object[] { "entity", new[] { "entityframework", "microsoft.entityframeworkcore" } }; + yield return new object[] { "json", new[] { "newtonsoft.json" } }; + yield return new object[] { "logging", new[] { "microsoft.extensions.logging" } }; + yield return new object[] { "aut", new[] { "autofac", "automapper" } }; + yield return new object[] { "mysql", new[] { "mysql.data", "mysqlconnector" } }; + yield return new object[] { "redi", new[] { "stackexchange.redis" } }; + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Relevancy/V3RelevancyFunctionalTests.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Relevancy/V3RelevancyFunctionalTests.cs new file mode 100644 index 000000000..a7898d168 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Relevancy/V3RelevancyFunctionalTests.cs @@ -0,0 +1,97 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class V3RelevancyFunctionalTests : NuGetSearchFunctionalTestBase + { + public V3RelevancyFunctionalTests(CommonFixture fixture, ITestOutputHelper testOutputHelper) + : base(fixture, testOutputHelper) + { + } + + [RelevancyTheory] + [MemberData(nameof(EnsureFirstResultsData))] + public async Task EnsureFirstResults(string searchTerm, string[] expectedFirstResults) + { + var results = await SearchAsync(searchTerm, take: 10); + + Assert.True(results.Count > expectedFirstResults.Length); + + for (var i = 0; i < expectedFirstResults.Length; i++) + { + Assert.True(expectedFirstResults[i] == results[i], $"Expected result '{expectedFirstResults[i]}' at index #{i} for query '{searchTerm}'"); + } + } + + public static IEnumerable EnsureFirstResultsData() + { + // Test that common queries have the most frequently selected results at the top. + // These results were determined using the "BrowserSearchPage" and "BrowserSearchSelection" metrics + // on the Gallery's Application Insights telemetry. + yield return new object[] { "newtonsoft.json", new[] { "newtonsoft.json" } }; + yield return new object[] { "newtonsoft", new[] { "newtonsoft.json" } }; + yield return new object[] { "json.net", new[] { "json.net" } }; + yield return new object[] { "json", new[] { "newtonsoft.json" } }; + yield return new object[] { "n", new[] { "newtonsoft.json" } }; + + yield return new object[] { "tags:\"aws-sdk-v3\"", new[] { "awssdk.core", "awssdk.s3" } }; + + yield return new object[] { "entityframework", new[] { "entityframework" } }; + yield return new object[] { "entity framework", new[] { "microsoft.entityframeworkcore" } }; + yield return new object[] { "EntityFrameworkCore", new[] { "microsoft.entityframeworkcore" } }; + yield return new object[] { "microsoft.entityframeworkcore", new[] { "microsoft.entityframeworkcore" } }; + yield return new object[] { "mysql", new[] { "mysql.data" } }; + + yield return new object[] { "microsoft.aspnetcore.app", new[] { "microsoft.aspnetcore.app" } }; + yield return new object[] { "microsoft.extensions.logging", new[] { "microsoft.extensions.logging" } }; + + yield return new object[] { "xunit", new[] { "xunit" } }; + yield return new object[] { "nunit", new[] { "nunit" } }; + yield return new object[] { "dapper", new[] { "dapper" } }; + yield return new object[] { "log4net", new[] { "log4net" } }; + yield return new object[] { "automapper", new[] { "automapper" } }; + yield return new object[] { "csv", new[] { "csvhelper" } }; + yield return new object[] { "bootstrap", new[] { "bootstrap" } }; + yield return new object[] { "moq", new[] { "moq" } }; + yield return new object[] { "serilog", new[] { "serilog" } }; + yield return new object[] { "redis", new[] { "stackexchange.redis", "microsoft.extensions.caching.redis" } }; + + // These tests were based off of external and internal feedback about exact match being first. + // https://github.com/NuGet/NuGetGallery/issues/7463 + yield return new object[] { "system.text.json", new[] { "system.text.json" } }; + yield return new object[] { "Westwind.AspNetCore.Markdown", new[] { "westwind.aspnetcore.markdown" } }; + + // This is currently a counter-example of the exact match case. For now, we don't exact match this package + // to the top. + yield return new object[] { "entity", new[] { "microsoft.entityframeworkcore" } }; + } + + [RelevancyTheory] + [MemberData(nameof(EnsureTopResultsData))] + public async Task EnsureTopResults(string searchTerm, string[] expectedTopResults) + { + var results = await SearchAsync(searchTerm, take: 10); + + Assert.True(results.Count > expectedTopResults.Length); + + foreach (var expectedTopResult in expectedTopResults) + { + Assert.True(results.Contains(expectedTopResult), $"Expected result '{expectedTopResult}' for query '{searchTerm}'"); + } + } + + public static IEnumerable EnsureTopResultsData() + { + // The following were chosen arbitrarily without telemetry. + yield return new object[] { "Microsoft.Extensions", new[] { "microsoft.extensions.logging", "microsoft.extensions.configuration", "microsoft.extensions.dependencyinjection" } }; + yield return new object[] { "mvc", new[] { "microsoft.aspnet.mvc", "microsoft.aspnetcore.mvc" } }; + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/AnalysisFactAttribute.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/AnalysisFactAttribute.cs new file mode 100644 index 000000000..9112cfa24 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/AnalysisFactAttribute.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class AnalysisFactAttribute : FactAttribute + { + public AnalysisFactAttribute() + { + if (!AzureSearchConfiguration.Create().TestSettings.RunAzureSearchAnalysisTests) + { + Skip = "Azure Search Analyzer tests are disabled"; + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/AnalysisTheoryAttribute.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/AnalysisTheoryAttribute.cs new file mode 100644 index 000000000..8f3666c53 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/AnalysisTheoryAttribute.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class AnalysisTheoryAttribute : TheoryAttribute + { + public AnalysisTheoryAttribute() + { + if (!AzureSearchConfiguration.Create().TestSettings.RunAzureSearchAnalysisTests) + { + Skip = "Azure Search Analyzer tests are disabled"; + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/AzureSearchConfiguration.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/AzureSearchConfiguration.cs new file mode 100644 index 000000000..e183065ce --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/AzureSearchConfiguration.cs @@ -0,0 +1,107 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using BasicSearchTests.FunctionalTests.Core; +using Newtonsoft.Json; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class AzureSearchConfiguration + { + private static AzureSearchConfiguration _configuration = null; + private static readonly SemaphoreSlim _semaphore = new SemaphoreSlim(1); + + public TestSettingsConfiguration TestSettings { get; set; } + + [JsonProperty] + public string Slot { get; set; } + + public string AzureSearchAppServiceUrl => "staging".Equals(Slot ?? "", StringComparison.OrdinalIgnoreCase) + ? TestSettings.AzureSearchAppServiceStagingUrl + : TestSettings.AzureSearchAppServiceProductionUrl; + + public static async Task CreateAsync() + { + if (_configuration != null) + { + return _configuration; + } + + try + { + await _semaphore.WaitAsync(); + + if (_configuration != null) + { + return _configuration; + } + + _configuration = CreateInternal(); + } + finally + { + _semaphore.Release(); + } + + return _configuration; + } + + public static AzureSearchConfiguration Create() + { + return CreateAsync().Result; + } + + private static AzureSearchConfiguration CreateInternal() + { + try + { + var configurationFilePath = EnvironmentSettings.ConfigurationFilePath; + var configurationString = File.ReadAllText(configurationFilePath); + var result = JsonConvert.DeserializeObject(configurationString); + return result; + } + catch (ArgumentException ae) + { + throw new ArgumentException( + $"No configuration file was specified! Set the '{EnvironmentSettings.ConfigurationFilePathVariableName}' environment variable to the path to a JSON configuration file.", + ae); + } + catch (Exception e) + { + throw new ArgumentException( + $"Unable to load the JSON configuration file. " + + $"Make sure the JSON configuration file exists at the path specified by the '{EnvironmentSettings.ConfigurationFilePathVariableName}' " + + $"and that it is a valid JSON file containing all required configuration.", + e); + } + } + + public class TestSettingsConfiguration + { + [JsonProperty] + public bool RunAzureSearchAnalysisTests { get; set; } + + [JsonProperty] + public bool RunAzureSearchRelevancyTests { get; set; } + + [JsonProperty] + public string AzureSearchIndexName { get; set; } + + [JsonProperty] + public string AzureSearchIndexUrl { get; set; } + + [JsonProperty] + public string AzureSearchIndexAdminApiKey { get; set; } + + [JsonProperty] + public string AzureSearchAppServiceProductionUrl { get; set; } + + [JsonProperty] + public string AzureSearchAppServiceStagingUrl { get; set; } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/AzureSearchIndexFunctionalTestBase.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/AzureSearchIndexFunctionalTestBase.cs new file mode 100644 index 000000000..ef5249cbc --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/AzureSearchIndexFunctionalTestBase.cs @@ -0,0 +1,62 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using BasicSearchTests.FunctionalTests.Core; +using Newtonsoft.Json; +using Xunit; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + /// + /// Base for functional tests that use Azure Search index APIs. + /// See: https://docs.microsoft.com/en-us/rest/api/searchservice/index-operations + /// + public class AzureSearchIndexFunctionalTestBase : BaseFunctionalTests, IClassFixture + { + public AzureSearchIndexFunctionalTestBase(CommonFixture fixture) + : base(fixture.AzureSearchConfiguration.TestSettings.AzureSearchIndexUrl) + { + Fixture = fixture ?? throw new ArgumentNullException(nameof(fixture)); + Client.DefaultRequestHeaders.Add("Api-Key", Fixture.AzureSearchConfiguration.TestSettings.AzureSearchIndexAdminApiKey); + } + + protected CommonFixture Fixture { get; private set; } + + protected async Task> AnalyzeAsync(string analyzer, string text) + { + var jsonContent = JsonConvert.SerializeObject(new + { + analyzer, + text + }); + + var index = Fixture.AzureSearchConfiguration.TestSettings.AzureSearchIndexName; + var requestUri = $"/indexes/{index}/analyze?api-version=2017-11-11"; + var content = new StringContent(jsonContent, Encoding.UTF8, "application/json"); + + var response = await Client.PostAsync(requestUri, content); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(); + var result = JsonConvert.DeserializeObject(json); + + return result?.Tokens.Select(t => t.Token).ToList(); + } + + private class AnalyzeResult + { + public IReadOnlyList Tokens { get; set; } + } + + private class AnalyzeResultToken + { + public string Token { get; set; } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/CommonFixture.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/CommonFixture.cs new file mode 100644 index 000000000..e72f571c4 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/CommonFixture.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Threading.Tasks; +using Xunit; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class CommonFixture : IAsyncLifetime + { + public AzureSearchConfiguration AzureSearchConfiguration { get; private set; } + + public async Task InitializeAsync() + { + AzureSearchConfiguration = await AzureSearchConfiguration.CreateAsync(); + } + + public Task DisposeAsync() => Task.CompletedTask; + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/NuGetSearchFunctionalTestBase.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/NuGetSearchFunctionalTestBase.cs new file mode 100644 index 000000000..d490daa86 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/NuGetSearchFunctionalTestBase.cs @@ -0,0 +1,126 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using BasicSearchTests.FunctionalTests.Core; +using BasicSearchTests.FunctionalTests.Core.Models; +using BasicSearchTests.FunctionalTests.Core.TestSupport; +using Newtonsoft.Json; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + /// + /// Base for functional tests on the NuGet search service. + /// + public class NuGetSearchFunctionalTestBase : BaseFunctionalTests, IClassFixture + { + private ITestOutputHelper _testOutputHelper; + + public NuGetSearchFunctionalTestBase(CommonFixture fixture, ITestOutputHelper testOutputHelper) + : base(fixture.AzureSearchConfiguration.AzureSearchAppServiceUrl) + { + Fixture = fixture ?? throw new ArgumentNullException(nameof(fixture)); + _testOutputHelper = testOutputHelper ?? throw new ArgumentNullException(nameof(testOutputHelper)); + _testOutputHelper.WriteLine($"Running tests against: {fixture.AzureSearchConfiguration.AzureSearchAppServiceUrl}"); + } + + protected CommonFixture Fixture { get; private set; } + + protected async Task> AutocompleteAsync( + string query, + int? skip = 0, + int? take = 20, + bool includePrerelease = true, + bool includeSemVer2 = true) + { + var results = await AutocompleteAsync(new AutocompleteBuilder() + { + Query = query, + Skip = skip, + Take = take, + Prerelease = includePrerelease, + IncludeSemVer2 = includeSemVer2, + }); + + var ids = results.Data.Select(t => t.ToLowerInvariant()).ToList(); + + _testOutputHelper.WriteLine("Got IDs:"); + for (var i = 0; i < ids.Count; i++) + { + _testOutputHelper.WriteLine($"{i + 1}. {ids[i]}"); + } + + return ids; + } + + /// + /// Queries the NuGet Search API. + /// See: https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource#search-for-packages + /// + /// The search terms to filter packages. + /// Whether prerelease results should be included. + /// Whether semver2 results should be included. + /// The package ids' that matches the query, lowercased. + protected async Task> SearchAsync( + string query, + int? skip = 0, + int? take = 20, + bool includePrerelease = true, + bool includeSemVer2 = true) + { + var results = await V3SearchAsync(new V3SearchBuilder() + { + Query = query, + Skip = skip, + Take = take, + Prerelease = includePrerelease, + IncludeSemVer2 = includeSemVer2 + }); + + var ids = results.Data.Select(t => t.Id.ToLowerInvariant()).ToList(); + + _testOutputHelper.WriteLine("Got IDs:"); + for (var i = 0; i < ids.Count; i++) + { + _testOutputHelper.WriteLine($"{i + 1}. {ids[i]}"); + } + + return ids; + } + + protected async Task V2SearchAsync(V2SearchBuilder searchBuilder) + { + return await SearchAsync(searchBuilder); + } + + protected async Task V3SearchAsync(V3SearchBuilder searchBuilder) + { + return await SearchAsync(searchBuilder); + } + + protected async Task AutocompleteAsync(AutocompleteBuilder searchBuilder) + { + return await SearchAsync(searchBuilder); + } + + private async Task SearchAsync(QueryBuilder searchBuilder) + { + var queryUrl = searchBuilder.RequestUri; + _testOutputHelper.WriteLine($"Fetching: {queryUrl}"); + using (var response = await Client.GetAsync(queryUrl)) + { + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(); + var result = JsonConvert.DeserializeObject(json); + + return result; + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/RelevancyFactAttribute.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/RelevancyFactAttribute.cs new file mode 100644 index 000000000..7aa2e8449 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/RelevancyFactAttribute.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class RelevancyFactAttribute : FactAttribute + { + public RelevancyFactAttribute() + { + if (!AzureSearchConfiguration.Create().TestSettings.RunAzureSearchRelevancyTests) + { + Skip = "Azure search Relevancy tests are disabled"; + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/RelevancyTheoryAttribute.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/RelevancyTheoryAttribute.cs new file mode 100644 index 000000000..63ab3e1a4 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/RelevancyTheoryAttribute.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public class RelevancyTheoryAttribute : TheoryAttribute + { + public RelevancyTheoryAttribute() + { + if (!AzureSearchConfiguration.Create().TestSettings.RunAzureSearchRelevancyTests) + { + Skip = "Azure search Relevancy tests are disabled"; + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/TestUtilities.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/TestUtilities.cs new file mode 100644 index 000000000..48bfcf4d3 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/TestUtilities.cs @@ -0,0 +1,37 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public static class TestUtilities + { + public static bool IsPrerelease(string original) + { + if (!NuGetVersion.TryParse(original, out var nugetVersion)) + { + return false; + } + + return nugetVersion.IsPrerelease; + } + + /// + /// Check for package version to be a SemVer2. This method only tests the version supplied. + /// The package can still be SemVer2 if its dependency is SemVer2, however this test only tests for the + /// provided version. + /// + /// Version string + /// True if the provided string is SemVer2, false otherwise + public static bool IsSemVer2(string original) + { + if (!NuGetVersion.TryParse(original, out var nugetVersion)) + { + return false; + } + + return nugetVersion.IsSemVer2; + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/TokenizationData.cs b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/TokenizationData.cs new file mode 100644 index 000000000..7da6c636a --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.FunctionalTests/Support/TokenizationData.cs @@ -0,0 +1,126 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; + +namespace NuGet.Services.AzureSearch.FunctionalTests +{ + public static class TokenizationData + { + public static readonly IEnumerable LowercasesTokens = ToMemberData(new Dictionary + { + { "hello", new[] { "hello"} }, + { "Hello", new[] { "hello" } }, + { "𠈓", new[] { "𠈓" } }, + }); + + public static readonly IEnumerable TrimsTokens = ToMemberData(new Dictionary + { + { " hello", new[] { "hello" } }, + { "\thello", new[] { "hello" } }, + { "\nhello", new[] { "hello" } }, + { "\rhello", new[] { "hello" } }, + + { "hello ", new[] { "hello" } }, + { "hello\t", new[] { "hello" } }, + { "hello\n", new[] { "hello" } }, + { "hello\r", new[] { "hello" } }, + }); + + private static readonly string TokenWith300Characters = new string('a', 300); + public static readonly IEnumerable TruncatesTokensAtLength300 = ToMemberData(new Dictionary + { + { TokenWith300Characters, new[] { TokenWith300Characters } }, + { TokenWith300Characters + 'z', new[] { TokenWith300Characters } } + }); + + public static readonly IEnumerable SplitsTokensAtLength300 = ToMemberData(new Dictionary + { + { TokenWith300Characters, new[] { TokenWith300Characters } }, + { TokenWith300Characters + 'z', new[] { TokenWith300Characters, "z" } } + }); + + public static readonly IEnumerable DoesNotSplitTokensOnSpecialCharacters = ToMemberData(new Dictionary + { + { "foo.bar", new[] { "foo.bar" } }, + { "foo-bar", new[] { "foo-bar" } }, + { "foo,bar", new[] { "foo,bar" } }, + { "foo;bar", new[] { "foo;bar" } }, + { "foo:bar", new[] { "foo:bar" } }, + { "foo'bar", new[] { "foo'bar" } }, + { "foo*bar", new[] { "foo*bar" } }, + { "foo#bar", new[] { "foo#bar" } }, + { "foo!bar", new[] { "foo!bar" } }, + { "foo~bar", new[] { "foo~bar" } }, + { "foo+bar", new[] { "foo+bar" } }, + { "foo(bar", new[] { "foo(bar" } }, + { "foo)bar", new[] { "foo)bar" } }, + { "foo[bar", new[] { "foo[bar" } }, + { "foo]bar", new[] { "foo]bar" } }, + { "foo{bar", new[] { "foo{bar" } }, + { "foo}bar", new[] { "foo}bar" } }, + { "foo_bar", new[] { "foo_bar" } }, + { "foo_𠈓_bar", new[] { "foo_𠈓_bar" } }, + }); + + public static readonly IEnumerable SplitsTokensOnSpecialCharactersAndLowercases = ToMemberData(new Dictionary + { + { "Foo.Bar", new[] { "foo", "bar" } }, + { "Foo-Bar", new[] { "foo", "bar" } }, + { "Foo,Bar", new[] { "foo", "bar" } }, + { "Foo;Bar", new[] { "foo", "bar" } }, + { "Foo:Bar", new[] { "foo", "bar" } }, + { "Foo'Bar", new[] { "foo", "bar" } }, + { "Foo*Bar", new[] { "foo", "bar" } }, + { "Foo#Bar", new[] { "foo", "bar" } }, + { "Foo!Bar", new[] { "foo", "bar" } }, + { "Foo~Bar", new[] { "foo", "bar" } }, + { "Foo+Bar", new[] { "foo", "bar" } }, + { "Foo(Bar", new[] { "foo", "bar" } }, + { "Foo)Bar", new[] { "foo", "bar" } }, + { "Foo[Bar", new[] { "foo", "bar" } }, + { "Foo]Bar", new[] { "foo", "bar" } }, + { "Foo{Bar", new[] { "foo", "bar" } }, + { "Foo}Bar", new[] { "foo", "bar" } }, + { "Foo_Bar", new[] { "foo", "bar" } }, + { "Foo_𠈓_bar", new[] { "foo", "𠈓", "bar" } }, + }); + + public static readonly IEnumerable LowercasesAndAddsTokensOnCasingAndNonAlphaNumeric = ToMemberData(new Dictionary + { + { "Microsoft.EntityFrameworkCore.SqlServer.Design", new[] { "microsoft", "entityframeworkcore", "entity", "framework", "core", "sqlserver", "sql", "server", "design" } }, + { "HelloWorld", new[] { "helloworld", "hello", "world" } }, + { "foo2bar", new[] { "foo2bar", "foo", "2", "bar" } }, + { "HTML", new[] { "html"} }, + { "HTMLThing", new[] { "htmlthing" } }, + { "HTMLThingA", new[] { "htmlthinga", "htmlthing", "a" } }, + { "HelloWorld𠈓Foo", new[] { "helloworld𠈓foo", "hello", "world𠈓foo" } }, + }); + + public static readonly IEnumerable AddsTokensOnNonAlphaNumericAndRemovesStopWords = ToMemberData(new Dictionary + { + { "a", new string[0] }, + { + "a an and are as at be but by hello for if in into is no not " + + "of on or such that the", + new[] + { + "hello" + } + }, + { + "Once upon a time, there was a little test-case!", + new[] + { + "once", "upon", "time", "little", "test", "case" + } + } + }); + + private static List ToMemberData(Dictionary data) + { + return data.Select(d => new object[] { d.Key, d.Value }).ToList(); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/DataSetComparerFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/DataSetComparerFacts.cs new file mode 100644 index 000000000..ba68d0f48 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/DataSetComparerFacts.cs @@ -0,0 +1,416 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Moq; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch +{ + public class DataSetComparerFacts + { + public class CompareOwners : Facts + { + public CompareOwners(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public void FindsAddedPackageIds() + { + var oldData = OwnersData("NuGet.Core: NuGet, Microsoft"); + var newData = OwnersData( + "NuGet.Core: NuGet, Microsoft", + "NuGet.Versioning: NuGet, Microsoft"); + + var changes = Target.CompareOwners(oldData, newData); + + var pair = Assert.Single(changes); + Assert.Equal("NuGet.Versioning", pair.Key); + Assert.Equal(new[] { "Microsoft", "NuGet" }, pair.Value); + + TelemetryService.Verify( + x => x.TrackOwnerSetComparison( + /*oldCount: */ 1, + /*newCount: */ 2, + /*changeCount: */ 1, + It.IsAny()), + Times.Once); + } + + [Fact] + public void FindsRemovedPackageIds() + { + var oldData = OwnersData( + "NuGet.Core: NuGet, Microsoft", + "NuGet.Versioning: NuGet, Microsoft"); + var newData = OwnersData("NuGet.Core: NuGet, Microsoft"); + + var changes = Target.CompareOwners(oldData, newData); + + var pair = Assert.Single(changes); + Assert.Equal("NuGet.Versioning", pair.Key); + Assert.Empty(pair.Value); + + TelemetryService.Verify( + x => x.TrackOwnerSetComparison( + /*oldCount: */ 2, + /*newCount: */ 1, + /*changeCount: */ 1, + It.IsAny()), + Times.Once); + } + + [Fact] + public void FindsAddedOwner() + { + var oldData = OwnersData("NuGet.Core: NuGet"); + var newData = OwnersData("NuGet.Core: NuGet, Microsoft"); + + var changes = Target.CompareOwners(oldData, newData); + + var pair = Assert.Single(changes); + Assert.Equal("NuGet.Core", pair.Key); + Assert.Equal(new[] { "Microsoft", "NuGet" }, pair.Value); + + TelemetryService.Verify( + x => x.TrackOwnerSetComparison( + /*oldCount: */ 1, + /*newCount: */ 1, + /*changeCount: */ 1, + It.IsAny()), + Times.Once); + } + + [Fact] + public void FindsRemovedOwner() + { + var oldData = OwnersData("NuGet.Core: NuGet, Microsoft"); + var newData = OwnersData("NuGet.Core: NuGet"); + + var changes = Target.CompareOwners(oldData, newData); + + var pair = Assert.Single(changes); + Assert.Equal("NuGet.Core", pair.Key); + Assert.Equal(new[] { "NuGet" }, pair.Value); + + TelemetryService.Verify( + x => x.TrackOwnerSetComparison( + /*oldCount: */ 1, + /*newCount: */ 1, + /*changeCount: */ 1, + It.IsAny()), + Times.Once); + } + + [Fact] + public void FindsOwnerWithChangedCase() + { + var oldData = OwnersData("NuGet.Core: NuGet, Microsoft"); + var newData = OwnersData("NuGet.Core: NuGet, microsoft"); + + var changes = Target.CompareOwners(oldData, newData); + + var pair = Assert.Single(changes); + Assert.Equal("NuGet.Core", pair.Key); + Assert.Equal(new[] { "microsoft", "NuGet" }, pair.Value); + + TelemetryService.Verify( + x => x.TrackOwnerSetComparison( + /*oldCount: */ 1, + /*newCount: */ 1, + /*changeCount: */ 1, + It.IsAny()), + Times.Once); + } + + [Fact] + public void FindsManyChangesAtOnce() + { + var oldData = OwnersData( + "NuGet.Core: NuGet, Microsoft", + "NuGet.Frameworks: NuGet", + "NuGet.Protocol: NuGet"); + var newData = OwnersData( + "NuGet.Core: NuGet, microsoft", + "NuGet.Versioning: NuGet", + "NuGet.Protocol: NuGet"); + + var changes = Target.CompareOwners(oldData, newData); + + Assert.Equal(3, changes.Count); + Assert.Equal(new[] { "NuGet.Core", "NuGet.Frameworks", "NuGet.Versioning" }, changes.Keys.ToArray()); + Assert.Equal(new[] { "microsoft", "NuGet" }, changes["NuGet.Core"]); + Assert.Empty(changes["NuGet.Frameworks"]); + Assert.Equal(new[] { "NuGet" }, changes["NuGet.Versioning"]); + + TelemetryService.Verify( + x => x.TrackOwnerSetComparison( + /*oldCount: */ 3, + /*newCount: */ 3, + /*changeCount: */ 3, + It.IsAny()), + Times.Once); + } + + [Fact] + public void FindsNoChanges() + { + var oldData = OwnersData( + "NuGet.Core: NuGet, Microsoft", + "NuGet.Versioning: NuGet, Microsoft"); + var newData = OwnersData( + "NuGet.Core: NuGet, Microsoft", + "NuGet.Versioning: NuGet, Microsoft"); + + var changes = Target.CompareOwners(oldData, newData); + + Assert.Empty(changes); + + TelemetryService.Verify( + x => x.TrackOwnerSetComparison( + /*oldCount: */ 2, + /*newCount: */ 2, + /*changeCount: */ 0, + It.IsAny()), + Times.Once); + } + } + + public class ComparePopularityTransfers : Facts + { + public ComparePopularityTransfers(ITestOutputHelper output) : base(output) + { + OldData = new PopularityTransferData(); + NewData = new PopularityTransferData(); + } + + public PopularityTransferData OldData { get; } + public PopularityTransferData NewData { get; } + + [Fact] + public void FindsNoChanges() + { + OldData.AddTransfer("PackageA", "PackageB"); + OldData.AddTransfer("PackageA", "PackageC"); + OldData.AddTransfer("Package1", "Package3"); + OldData.AddTransfer("Package1", "Package2"); + + NewData.AddTransfer("PackageA", "PackageC"); + NewData.AddTransfer("PackageA", "PackageB"); + NewData.AddTransfer("Package1", "Package2"); + NewData.AddTransfer("Package1", "Package3"); + + var changes = Target.ComparePopularityTransfers(OldData, NewData); + + Assert.Empty(changes); + + TelemetryService.Verify( + x => x.TrackPopularityTransfersSetComparison( + /*oldCount: */ 2, + /*newCount: */ 2, + /*changeCount: */ 0, + It.IsAny()), + Times.Once); + } + + [Fact] + public void FindsAddedTransfers() + { + OldData.AddTransfer("PackageA", "PackageB"); + OldData.AddTransfer("PackageA", "PackageC"); + + NewData.AddTransfer("PackageA", "PackageB"); + NewData.AddTransfer("PackageA", "PackageC"); + NewData.AddTransfer("Package1", "Package2"); + NewData.AddTransfer("Package1", "Package3"); + + var changes = Target.ComparePopularityTransfers(OldData, NewData); + + var pair = Assert.Single(changes); + Assert.Equal("Package1", pair.Key); + Assert.Equal(new[] { "Package2", "Package3" }, pair.Value); + + TelemetryService.Verify( + x => x.TrackPopularityTransfersSetComparison( + /*oldCount: */ 1, + /*newCount: */ 2, + /*changeCount: */ 1, + It.IsAny()), + Times.Once); + } + + [Fact] + public void FindsRemovedTransfers() + { + OldData.AddTransfer("PackageA", "PackageB"); + OldData.AddTransfer("PackageA", "PackageC"); + OldData.AddTransfer("Package1", "Package2"); + OldData.AddTransfer("Package1", "Package3"); + + NewData.AddTransfer("PackageA", "PackageB"); + NewData.AddTransfer("PackageA", "PackageC"); + + var changes = Target.ComparePopularityTransfers(OldData, NewData); + + var pair = Assert.Single(changes); + Assert.Equal("Package1", pair.Key); + Assert.Empty(pair.Value); + + TelemetryService.Verify( + x => x.TrackPopularityTransfersSetComparison( + /*oldCount: */ 2, + /*newCount: */ 1, + /*changeCount: */ 1, + It.IsAny()), + Times.Once); + } + + [Fact] + public void FindsAddedToPackage() + { + OldData.AddTransfer("PackageA", "PackageB"); + + NewData.AddTransfer("PackageA", "PackageB"); + NewData.AddTransfer("PackageA", "PackageC"); + + var changes = Target.ComparePopularityTransfers(OldData, NewData); + + var pair = Assert.Single(changes); + Assert.Equal("PackageA", pair.Key); + Assert.Equal(new[] { "PackageB", "PackageC" }, pair.Value); + + TelemetryService.Verify( + x => x.TrackPopularityTransfersSetComparison( + /*oldCount: */ 1, + /*newCount: */ 1, + /*changeCount: */ 1, + It.IsAny()), + Times.Once); + } + + [Fact] + public void FindsRemovedToPackage() + { + OldData.AddTransfer("PackageA", "PackageB"); + OldData.AddTransfer("PackageA", "PackageC"); + + NewData.AddTransfer("PackageA", "PackageB"); + + var changes = Target.ComparePopularityTransfers(OldData, NewData); + + var pair = Assert.Single(changes); + Assert.Equal("PackageA", pair.Key); + Assert.Equal(new[] { "PackageB" }, pair.Value); + + TelemetryService.Verify( + x => x.TrackPopularityTransfersSetComparison( + /*oldCount: */ 1, + /*newCount: */ 1, + /*changeCount: */ 1, + It.IsAny()), + Times.Once); + } + + [Fact] + public void IgnoresCaseChanges() + { + OldData.AddTransfer("PackageA", "packageb"); + OldData.AddTransfer("PackageA", "PackageC"); + + NewData.AddTransfer("packagea", "PACKAGEB"); + NewData.AddTransfer("PackageA", "packageC"); + + var changes = Target.ComparePopularityTransfers(OldData, NewData); + + Assert.Empty(changes); + + TelemetryService.Verify( + x => x.TrackPopularityTransfersSetComparison( + /*oldCount: */ 1, + /*newCount: */ 1, + /*changeCount: */ 0, + It.IsAny()), + Times.Once); + } + + [Fact] + public void FindsManyChangesAtOnce() + { + OldData.AddTransfer("Package1", "PackageA"); + OldData.AddTransfer("Package1", "PackageB"); + OldData.AddTransfer("Package2", "PackageC"); + OldData.AddTransfer("Package3", "PackageD"); + + NewData.AddTransfer("Package1", "PackageA"); + NewData.AddTransfer("Package1", "PackageE"); + NewData.AddTransfer("Package4", "PackageC"); + NewData.AddTransfer("Package3", "Packaged"); + + var changes = Target.ComparePopularityTransfers(OldData, NewData); + + Assert.Equal(3, changes.Count); + Assert.Equal(new[] { "Package1", "Package2", "Package4" }, changes.Keys.ToArray()); + Assert.Equal(new[] { "PackageA", "PackageE" }, changes["Package1"]); + Assert.Empty(changes["Package2"]); + Assert.Equal(new[] { "PackageC" }, changes["Package4"]); + + TelemetryService.Verify( + x => x.TrackPopularityTransfersSetComparison( + /*oldCount: */ 3, + /*newCount: */ 3, + /*changeCount: */ 3, + It.IsAny()), + Times.Once); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + TelemetryService = new Mock(); + Logger = output.GetLogger(); + + Target = new DataSetComparer( + TelemetryService.Object, + Logger); + } + + public Mock TelemetryService { get; } + public RecordingLogger Logger { get; } + public DataSetComparer Target { get; } + + /// + /// A helper to turn lines formatted like this "PackageId: OwnerA, OwnerB" into package ID to owners + /// dictionary. + /// + public SortedDictionary> OwnersData(params string[] lines) + { + var builder = new PackageIdToOwnersBuilder(Logger); + ParseData(lines, builder.Add); + return builder.GetResult(); + } + + private void ParseData(string[] lines, Action> add) + { + foreach (var line in lines) + { + var pieces = line.Split(new[] { ':' }, 2); + var key = pieces[0].Trim(); + var values = pieces[1] + .Split(',') + .Select(x => x.Trim()) + .Where(x => x.Length > 0) + .ToList(); + + add(key, values); + } + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/DownloadSetComparerFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/DownloadSetComparerFacts.cs new file mode 100644 index 000000000..875e77a90 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/DownloadSetComparerFacts.cs @@ -0,0 +1,288 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch +{ + public class DownloadSetComparerFacts + { + public class Compare : Facts + { + public Compare(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public void RejectsEmptyNewData() + { + var oldData = new DownloadData(); + oldData.SetDownloadCount(IdA, V1, 5); + var newData = new DownloadData(); + + var ex = Assert.Throws(() => Target.Compare(oldData, newData)); + Assert.Equal("The new data should not be empty.", ex.Message); + } + + [Theory] + [InlineData(9, false)] + [InlineData(10, false)] + [InlineData(11, true)] + [InlineData(12, true)] + public void DetectsTooManyDecreases(int decreases, bool tooMany) + { + Config.MaxDownloadCountDecreases = 10; + + var oldData = new DownloadData(); + oldData.SetDownloadCount($"NuGet.Frameworks", "1.0.0-alpha", 1); + for (var i = 0; i < decreases; i++) + { + oldData.SetDownloadCount($"NuGet.Versioning.{i}", "1.0.0-alpha", 1); + } + + var newData = new DownloadData(); + newData.SetDownloadCount($"NuGet.Frameworks", "1.0.0-alpha", 1); + + if (tooMany) + { + var ex = Assert.Throws(() => Target.Compare(oldData, newData)); + Assert.Equal("Too many download count decreases are occurring.", ex.Message); + Assert.Contains($"There are {decreases} package versions with download count decreases.", Logger.Messages); + } + else + { + var delta = Target.Compare(oldData, newData); + Assert.Equal(decreases, delta.Count); + } + } + + [Fact] + public void DetectsNoChange() + { + var oldData = new DownloadData(); + oldData.SetDownloadCount(IdA, V1, 5); + oldData.SetDownloadCount(IdB, V1, 1); + var newData = new DownloadData(); + newData.SetDownloadCount(IdA, V1, 5); + newData.SetDownloadCount(IdB, V1, 1); + + var delta = Target.Compare(oldData, newData); + + Assert.Empty(delta); + VerifyDecreaseTelemetry(Times.Never()); + } + + [Fact] + public void DetectsIncreaseDueToRealIncrease() + { + var oldData = new DownloadData(); + oldData.SetDownloadCount(IdA, V1, 5); + oldData.SetDownloadCount(IdB, V1, 1); + var newData = new DownloadData(); + newData.SetDownloadCount(IdA, V1, 7); // Increase + newData.SetDownloadCount(IdB, V1, 1); // No change + + var delta = Target.Compare(oldData, newData); + + Assert.Equal(KeyValuePair.Create(IdA, 7L), Assert.Single(delta)); + VerifyDecreaseTelemetry(Times.Never()); + } + + [Fact] + public void DetectsIncreaseDueToAddedVersion() + { + var oldData = new DownloadData(); + oldData.SetDownloadCount(IdA, V1, 5); + var newData = new DownloadData(); + newData.SetDownloadCount(IdA, V1, 5); // No change + newData.SetDownloadCount(IdA, V2, 2); // Added + + var delta = Target.Compare(oldData, newData); + + Assert.Equal(KeyValuePair.Create(IdA, 7L), Assert.Single(delta)); + VerifyDecreaseTelemetry(Times.Never()); + } + + [Fact] + public void DetectsIncreaseDueToAddedId() + { + var oldData = new DownloadData(); + var newData = new DownloadData(); + newData.SetDownloadCount(IdA, V1, 5); // Added + newData.SetDownloadCount(IdA, V2, 2); // Added + + var delta = Target.Compare(oldData, newData); + + Assert.Equal(KeyValuePair.Create(IdA, 7L), Assert.Single(delta)); + VerifyDecreaseTelemetry(Times.Never()); + } + + [Fact] + public void DetectsDecreaseWhenTotalRemainsTheSame() + { + var oldData = new DownloadData(); + oldData.SetDownloadCount(IdA, V1, 7); + oldData.SetDownloadCount(IdA, V2, 1); + var newData = new DownloadData(); + newData.SetDownloadCount(IdA, V1, 5); + newData.SetDownloadCount(IdA, V2, 3); + + var delta = Target.Compare(oldData, newData); + + Assert.Empty(delta); + VerifyDecreaseTelemetry(Times.Once()); + TelemetryService.Verify( + x => x.TrackDownloadCountDecrease( + IdA, + V1, + true, + true, + 7, + true, + true, + 5), + Times.Once); + } + + [Fact] + public void DetectsDecreaseDueToRealDecrease() + { + var oldData = new DownloadData(); + oldData.SetDownloadCount(IdA, V1, 7); + oldData.SetDownloadCount(IdA, V2, 1); + var newData = new DownloadData(); + newData.SetDownloadCount(IdA, V1, 5); + newData.SetDownloadCount(IdA, V2, 1); + + var delta = Target.Compare(oldData, newData); + + Assert.Equal(KeyValuePair.Create(IdA, 6L), Assert.Single(delta)); + VerifyDecreaseTelemetry(Times.Once()); + TelemetryService.Verify( + x => x.TrackDownloadCountDecrease( + IdA, + V1, + true, + true, + 7, + true, + true, + 5), + Times.Once); + } + + [Fact] + public void DetectsDecreaseDueToMissingVersion() + { + var oldData = new DownloadData(); + oldData.SetDownloadCount(IdA, V1, 7); + oldData.SetDownloadCount(IdA, V2, 1); + var newData = new DownloadData(); + newData.SetDownloadCount(IdA, V2, 1); + + var delta = Target.Compare(oldData, newData); + + Assert.Equal(KeyValuePair.Create(IdA, 1L), Assert.Single(delta)); + VerifyDecreaseTelemetry(Times.Once()); + TelemetryService.Verify( + x => x.TrackDownloadCountDecrease( + IdA, + V1, + true, + true, + 7, + true, + false, + 0), + Times.Once); + } + + [Fact] + public void DetectsDecreaseDueToMissingId() + { + var oldData = new DownloadData(); + oldData.SetDownloadCount(IdA, V1, 7); + oldData.SetDownloadCount(IdA, V2, 1); + oldData.SetDownloadCount(IdB, V1, 1); + var newData = new DownloadData(); + newData.SetDownloadCount(IdB, V1, 1); + + var delta = Target.Compare(oldData, newData); + + Assert.Equal(KeyValuePair.Create(IdA, 0L), Assert.Single(delta)); + VerifyDecreaseTelemetry(Times.Exactly(2)); + TelemetryService.Verify( + x => x.TrackDownloadCountDecrease( + IdA, + V1, + true, + true, + 7, + false, + false, + 0), + Times.Once); + TelemetryService.Verify( + x => x.TrackDownloadCountDecrease( + IdA, + V2, + true, + true, + 1, + false, + false, + 0), + Times.Once); + } + } + + public abstract class Facts + { + public const string IdA = "NuGet.Frameworks"; + public const string IdB = "NuGet.Versioning"; + public const string V1 = "1.0.0"; + public const string V2 = "2.0.0"; + + public Facts(ITestOutputHelper output) + { + TelemetryService = new Mock(); + Options = new Mock>(); + Logger = output.GetLogger(); + + Config = new Auxiliary2AzureSearchConfiguration(); + Options.Setup(x => x.Value).Returns(() => Config); + + Target = new DownloadSetComparer( + TelemetryService.Object, + Options.Object, + Logger); + } + + public Mock TelemetryService { get; } + public Mock> Options { get; } + public RecordingLogger Logger { get; } + public Auxiliary2AzureSearchConfiguration Config { get; } + public DownloadSetComparer Target { get; } + + public void VerifyDecreaseTelemetry(Times times) + { + TelemetryService.Verify( + x => x.TrackDownloadCountDecrease( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + times); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/Integration/PopularityTransferIntegrationTests.cs b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/Integration/PopularityTransferIntegrationTests.cs new file mode 100644 index 000000000..f367ecdd6 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/Integration/PopularityTransferIntegrationTests.cs @@ -0,0 +1,493 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.AzureSearch.Wrappers; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch.Integration +{ + public class PopularityTransferIntegrationTests + { + private readonly InMemoryCloudBlobClient _blobClient; + private readonly InMemoryCloudBlobContainer _auxilliaryContainer; + private readonly InMemoryCloudBlobContainer _storageContainer; + + private readonly Mock _searchOperations; + private readonly Mock _featureFlags; + private readonly Auxiliary2AzureSearchConfiguration _config; + private readonly AzureSearchJobDevelopmentConfiguration _developmentConfig; + private readonly Mock _telemetry; + private readonly UpdateDownloadsCommand _target; + + private readonly PopularityTransferData _newPopularityTransfers; + + private IndexBatch _indexedBatch; + + public PopularityTransferIntegrationTests(ITestOutputHelper output) + { + _featureFlags = new Mock(); + _telemetry = new Mock(); + + _config = new Auxiliary2AzureSearchConfiguration + { + AuxiliaryDataStorageContainer = "auxiliary-container", + EnablePopularityTransfers = true, + StorageContainer = "storage-container", + Scoring = new AzureSearchScoringConfiguration() + }; + + var options = new Mock>(); + options + .Setup(x => x.Value) + .Returns(_config); + + _developmentConfig = new AzureSearchJobDevelopmentConfiguration(); + var developmentOptions = new Mock>(); + developmentOptions + .Setup(x => x.Value) + .Returns(_developmentConfig); + + var auxiliaryConfig = new AuxiliaryDataStorageConfiguration + { + AuxiliaryDataStorageContainer = "auxiliary-container", + AuxiliaryDataStorageDownloadsPath = "downloads.json", + AuxiliaryDataStorageExcludedPackagesPath = "excludedPackages.json", + }; + + var auxiliaryOptions = new Mock>(); + auxiliaryOptions + .Setup(x => x.Value) + .Returns(auxiliaryConfig); + + _auxilliaryContainer = new InMemoryCloudBlobContainer(); + _storageContainer = new InMemoryCloudBlobContainer(); + + _blobClient = new InMemoryCloudBlobClient(); + _blobClient.Containers["auxiliary-container"] = _auxilliaryContainer; + _blobClient.Containers["storage-container"] = _storageContainer; + + var auxiliaryFileClient = new AuxiliaryFileClient( + _blobClient, + auxiliaryOptions.Object, + _telemetry.Object, + output.GetLogger()); + + _newPopularityTransfers = new PopularityTransferData(); + var databaseFetcher = new Mock(); + databaseFetcher + .Setup(x => x.GetPopularityTransfersAsync()) + .ReturnsAsync(_newPopularityTransfers); + + var downloadDataClient = new DownloadDataClient( + _blobClient, + options.Object, + _telemetry.Object, + output.GetLogger()); + + var popularityTransferDataClient = new PopularityTransferDataClient( + _blobClient, + options.Object, + _telemetry.Object, + output.GetLogger()); + + var versionListDataClient = new VersionListDataClient( + _blobClient, + options.Object, + output.GetLogger()); + + var downloadComparer = new DownloadSetComparer( + _telemetry.Object, + options.Object, + output.GetLogger()); + + var dataComparer = new DataSetComparer( + _telemetry.Object, + output.GetLogger()); + + var downloadTransferrer = new DownloadTransferrer( + dataComparer, + options.Object, + output.GetLogger()); + + var baseDocumentBuilder = new BaseDocumentBuilder(options.Object); + var searchDocumentBuilder = new SearchDocumentBuilder(baseDocumentBuilder); + var searchIndexActionBuilder = new SearchIndexActionBuilder( + versionListDataClient, + output.GetLogger()); + + _searchOperations = new Mock(); + _searchOperations + .Setup(x => x.IndexAsync(It.IsAny>())) + .Callback>(batch => + { + _indexedBatch = batch; + }) + .ReturnsAsync(new DocumentIndexResult()); + + var hijackIndexClient = new Mock(); + var searchIndexClient = new Mock(); + searchIndexClient + .Setup(x => x.Documents) + .Returns(_searchOperations.Object); + + var batchPusher = new BatchPusher( + searchIndexClient.Object, + hijackIndexClient.Object, + versionListDataClient, + options.Object, + developmentOptions.Object, + _telemetry.Object, + output.GetLogger()); + + Func batchPusherFactory = () => batchPusher; + + var time = new Mock(); + + _featureFlags.Setup(x => x.IsPopularityTransferEnabled()).Returns(true); + + _target = new UpdateDownloadsCommand( + auxiliaryFileClient, + databaseFetcher.Object, + downloadDataClient, + downloadComparer, + downloadTransferrer, + popularityTransferDataClient, + searchDocumentBuilder, + searchIndexActionBuilder, + batchPusherFactory, + time.Object, + _featureFlags.Object, + options.Object, + _telemetry.Object, + output.GetLogger()); + } + + [Fact] + public async Task FirstPopularityTransferChangesDownloads() + { + SetExcludedPackagesJson("{}"); + + AddVersionList("A", "1.0.0"); + AddVersionList("B", "1.0.0"); + + SetOldDownloadsJson(@" +{ + ""A"": { ""1.0.0"": 100 }, + ""B"": { ""1.0.0"": 1 } +}"); + SetNewDownloadsJson(@" +[ + [ ""A"", [ ""1.0.0"", 100 ] ], + [ ""B"", [ ""1.0.0"", 1 ] ], +]"); + + // Old: no rename + // New: A -> B rename + SetOldPopularityTransfersJson(@"{}"); + _newPopularityTransfers.AddTransfer("A", "B"); + + _config.Scoring.PopularityTransfer = 0.5; + + await _target.ExecuteAsync(); + + Assert.NotNull(_indexedBatch); + var actions = _indexedBatch.Actions.OrderBy(x => x.Document.Key).ToList(); + Assert.Equal(8, actions.Count); + + VerifyUpdateDownloadCountAction("A", 50, actions[0]); + VerifyUpdateDownloadCountAction("A", 50, actions[1]); + VerifyUpdateDownloadCountAction("A", 50, actions[2]); + VerifyUpdateDownloadCountAction("A", 50, actions[3]); + VerifyUpdateDownloadCountAction("B", 51, actions[4]); + VerifyUpdateDownloadCountAction("B", 51, actions[5]); + VerifyUpdateDownloadCountAction("B", 51, actions[6]); + VerifyUpdateDownloadCountAction("B", 51, actions[7]); + } + + [Fact] + public async Task NewPopularityTransferChangesDownloads() + { + SetExcludedPackagesJson("{}"); + + AddVersionList("A", "1.0.0"); + AddVersionList("B", "1.0.0"); + AddVersionList("C", "1.0.0"); + AddVersionList("D", "1.0.0"); + + SetOldDownloadsJson(@" +{ + ""A"": { ""1.0.0"": 100 }, + ""B"": { ""1.0.0"": 50 }, + ""C"": { ""1.0.0"": 20 }, + ""D"": { ""1.0.0"": 1 } +}"); + SetNewDownloadsJson(@" +[ + [ ""A"", [ ""1.0.0"", 100 ] ], + [ ""B"", [ ""1.0.0"", 50 ] ], + [ ""C"", [ ""1.0.0"", 20 ] ], + [ ""D"", [ ""1.0.0"", 1 ] ] +]"); + + // Old: A -> B rename + // New: A -> B, C -> D rename + SetOldPopularityTransfersJson(@"{ ""A"": [ ""B"" ] }"); + _newPopularityTransfers.AddTransfer("A", "B"); + _newPopularityTransfers.AddTransfer("C", "D"); + + _config.Scoring.PopularityTransfer = 0.5; + + await _target.ExecuteAsync(); + + Assert.NotNull(_indexedBatch); + var actions = _indexedBatch.Actions.OrderBy(x => x.Document.Key).ToList(); + Assert.Equal(8, actions.Count); + + VerifyUpdateDownloadCountAction("C", 10, actions[0]); + VerifyUpdateDownloadCountAction("C", 10, actions[1]); + VerifyUpdateDownloadCountAction("C", 10, actions[2]); + VerifyUpdateDownloadCountAction("C", 10, actions[3]); + VerifyUpdateDownloadCountAction("D", 11, actions[4]); + VerifyUpdateDownloadCountAction("D", 11, actions[5]); + VerifyUpdateDownloadCountAction("D", 11, actions[6]); + VerifyUpdateDownloadCountAction("D", 11, actions[7]); + } + + [Fact] + public async Task UpdatedPopularityTransferChangesDownloads() + { + SetExcludedPackagesJson("{}"); + + AddVersionList("A", "1.0.0"); + AddVersionList("B", "1.0.0"); + AddVersionList("C", "1.0.0"); + + SetOldDownloadsJson(@" +{ + ""A"": { ""1.0.0"": 100 }, + ""B"": { ""1.0.0"": 20 }, + ""C"": { ""1.0.0"": 1 }, +}"); + SetNewDownloadsJson(@" +[ + [ ""A"", [ ""1.0.0"", 100 ] ], + [ ""B"", [ ""1.0.0"", 20 ] ], + [ ""C"", [ ""1.0.0"", 1 ] ], +]"); + + // Old: A -> B rename + // New: A -> C rename + SetOldPopularityTransfersJson(@"{ ""A"": [ ""B"" ] }"); + _newPopularityTransfers.AddTransfer("A", "C"); + + _config.Scoring.PopularityTransfer = 0.5; + + await _target.ExecuteAsync(); + + Assert.NotNull(_indexedBatch); + var actions = _indexedBatch.Actions.OrderBy(x => x.Document.Key).ToList(); + Assert.Equal(12, actions.Count); + + VerifyUpdateDownloadCountAction("A", 50, actions[0]); + VerifyUpdateDownloadCountAction("A", 50, actions[1]); + VerifyUpdateDownloadCountAction("A", 50, actions[2]); + VerifyUpdateDownloadCountAction("A", 50, actions[3]); + VerifyUpdateDownloadCountAction("B", 20, actions[4]); + VerifyUpdateDownloadCountAction("B", 20, actions[5]); + VerifyUpdateDownloadCountAction("B", 20, actions[6]); + VerifyUpdateDownloadCountAction("B", 20, actions[7]); + VerifyUpdateDownloadCountAction("C", 51, actions[8]); + VerifyUpdateDownloadCountAction("C", 51, actions[9]); + VerifyUpdateDownloadCountAction("C", 51, actions[10]); + VerifyUpdateDownloadCountAction("C", 51, actions[11]); + } + + [Fact] + public async Task ReverseTransferChangesDownloads() + { + SetExcludedPackagesJson("{}"); + + AddVersionList("A", "1.0.0"); + AddVersionList("B", "1.0.0"); + + SetOldDownloadsJson(@" +{ + ""A"": { ""1.0.0"": 100 }, + ""B"": { ""1.0.0"": 20 } +}"); + SetNewDownloadsJson(@" +[ + [ ""A"", [ ""1.0.0"", 100 ] ], + [ ""B"", [ ""1.0.0"", 20 ] ] +]"); + + // Old: A -> B rename + // New: B -> A rename + SetOldPopularityTransfersJson(@"{ ""A"": [ ""B"" ] }"); + _newPopularityTransfers.AddTransfer("B", "A"); + + _config.Scoring.PopularityTransfer = 0.5; + + await _target.ExecuteAsync(); + + Assert.NotNull(_indexedBatch); + var actions = _indexedBatch.Actions.OrderBy(x => x.Document.Key).ToList(); + Assert.Equal(8, actions.Count); + + VerifyUpdateDownloadCountAction("A", 110, actions[0]); + VerifyUpdateDownloadCountAction("A", 110, actions[1]); + VerifyUpdateDownloadCountAction("A", 110, actions[2]); + VerifyUpdateDownloadCountAction("A", 110, actions[3]); + VerifyUpdateDownloadCountAction("B", 10, actions[4]); + VerifyUpdateDownloadCountAction("B", 10, actions[5]); + VerifyUpdateDownloadCountAction("B", 10, actions[6]); + VerifyUpdateDownloadCountAction("B", 10, actions[7]); + } + + [Fact] + public async Task DisablingPopularityTransferConfigRemovesTransfers() + { + SetExcludedPackagesJson("{}"); + + AddVersionList("A", "1.0.0"); + AddVersionList("B", "1.0.0"); + AddVersionList("C", "1.0.0"); + + SetOldDownloadsJson(@" +{ + ""A"": { ""1.0.0"": 100 }, + ""B"": { ""1.0.0"": 20 }, + ""C"": { ""1.0.0"": 1 } +}"); + SetNewDownloadsJson(@" +[ + [ ""A"", [ ""1.0.0"", 100 ] ], + [ ""B"", [ ""1.0.0"", 20 ] ], + [ ""C"", [ ""1.0.0"", 1 ] ] +]"); + + // Old: A -> B rename + // New: A -> B rename + SetOldPopularityTransfersJson(@"{ ""A"": [ ""B"" ] }"); + _newPopularityTransfers.AddTransfer("A", "B"); + + _config.EnablePopularityTransfers = false; + _config.Scoring.PopularityTransfer = 0.5; + + await _target.ExecuteAsync(); + + Assert.NotNull(_indexedBatch); + var actions = _indexedBatch.Actions.OrderBy(x => x.Document.Key).ToList(); + Assert.Equal(8, actions.Count); + + VerifyUpdateDownloadCountAction("A", 100, actions[0]); + VerifyUpdateDownloadCountAction("A", 100, actions[1]); + VerifyUpdateDownloadCountAction("A", 100, actions[2]); + VerifyUpdateDownloadCountAction("A", 100, actions[3]); + VerifyUpdateDownloadCountAction("B", 20, actions[4]); + VerifyUpdateDownloadCountAction("B", 20, actions[5]); + VerifyUpdateDownloadCountAction("B", 20, actions[6]); + VerifyUpdateDownloadCountAction("B", 20, actions[7]); + } + + [Fact] + public async Task DisablingPopularityTransferFeatureRemovesTransfers() + { + SetExcludedPackagesJson("{}"); + + AddVersionList("A", "1.0.0"); + AddVersionList("B", "1.0.0"); + AddVersionList("C", "1.0.0"); + + SetOldDownloadsJson(@" +{ + ""A"": { ""1.0.0"": 100 }, + ""B"": { ""1.0.0"": 20 }, + ""C"": { ""1.0.0"": 1 } +}"); + SetNewDownloadsJson(@" +[ + [ ""A"", [ ""1.0.0"", 100 ] ], + [ ""B"", [ ""1.0.0"", 20 ] ], + [ ""C"", [ ""1.0.0"", 1 ] ] +]"); + + // Old: A -> B rename + // New: A -> B rename + SetOldPopularityTransfersJson(@"{ ""A"": [ ""B"" ] }"); + _newPopularityTransfers.AddTransfer("A", "B"); + + _config.Scoring.PopularityTransfer = 0.5; + _featureFlags + .Setup(x => x.IsPopularityTransferEnabled()) + .Returns(false); + + await _target.ExecuteAsync(); + + Assert.NotNull(_indexedBatch); + var actions = _indexedBatch.Actions.OrderBy(x => x.Document.Key).ToList(); + Assert.Equal(8, actions.Count); + + VerifyUpdateDownloadCountAction("A", 100, actions[0]); + VerifyUpdateDownloadCountAction("A", 100, actions[1]); + VerifyUpdateDownloadCountAction("A", 100, actions[2]); + VerifyUpdateDownloadCountAction("A", 100, actions[3]); + VerifyUpdateDownloadCountAction("B", 20, actions[4]); + VerifyUpdateDownloadCountAction("B", 20, actions[5]); + VerifyUpdateDownloadCountAction("B", 20, actions[6]); + VerifyUpdateDownloadCountAction("B", 20, actions[7]); + } + + private void SetOldDownloadsJson(string json) + { + _storageContainer.Blobs["downloads/downloads.v2.json"] = new InMemoryCloudBlob(json); + } + + private void SetNewDownloadsJson(string json) + { + _auxilliaryContainer.Blobs["downloads.json"] = new InMemoryCloudBlob(json); + } + + private void SetExcludedPackagesJson(string json) + { + _auxilliaryContainer.Blobs["excludedPackages.json"] = new InMemoryCloudBlob(json); + } + + private void SetOldPopularityTransfersJson(string json) + { + _storageContainer.Blobs["popularity-transfers/popularity-transfers.v1.json"] + = new InMemoryCloudBlob(json); + } + + private void AddVersionList(string id, string version) + { + _storageContainer.Blobs[$"version-lists/{id.ToLowerInvariant()}.json"] = new InMemoryCloudBlob(@" +{ + ""VersionProperties"": { + """ + version + @""": { ""Listed"": true } + } +}"); + } + + private void VerifyUpdateDownloadCountAction( + string expectedId, + long expectedDownloads, + IndexAction action) + { + var document = action.Document as SearchDocument.UpdateDownloadCount; + + Assert.NotNull(document); + Assert.Equal(IndexActionType.Merge, action.ActionType); + Assert.StartsWith(expectedId, document.Key, StringComparison.OrdinalIgnoreCase); + Assert.Equal(expectedDownloads, document.TotalDownloadCount); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/UpdateDownloadsCommandFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/UpdateDownloadsCommandFacts.cs new file mode 100644 index 000000000..73cd8d477 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/UpdateDownloadsCommandFacts.cs @@ -0,0 +1,658 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.AzureSearch.Support; +using NuGet.Services.AzureSearch.Wrappers; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch +{ + public class UpdateDownloadsCommandFacts + { + public class ExecuteAsync : Facts + { + public ExecuteAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task PushesNothingWhenThereAreNoChanges() + { + await Target.ExecuteAsync(); + + VerifyCompletedTelemetry(JobOutcome.NoOp); + VerifyAllIdsAreProcessed(changeCount: 0); + IndexActionBuilder.Verify( + x => x.UpdateAsync( + It.IsAny(), + It.IsAny>()), + Times.Never); + BatchPusher.Verify(x => x.TryFinishAsync(), Times.Never); + BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never); + DownloadDataClient.Verify( + x => x.ReplaceLatestIndexedAsync(It.IsAny(), It.IsAny()), + Times.Never); + PopularityTransferDataClient.Verify( + x => x.ReplaceLatestIndexedAsync(It.IsAny(), It.IsAny()), + Times.Never); + } + + [Theory] + [InlineData(1, 7, 4, 1)] // 1, 2, ... 7 + 4 = 11 is greater than 10 so 7 is the batch size. + [InlineData(2, 8, 7, 1)] // 2, 4, ... 8 + 4 = 12 is greater than 10 so 8 is the batch size. + [InlineData(3, 9, 10, 0)] // 3, 6, 9 + 4 = 13 is greater than 10 so 9 is the batch size. + [InlineData(4, 8, 15, 0)] // 4, 8 + 4 = 12 is greater than 10 so 8 is the batch size. + public async Task RespectsAzureSearchBatchSize(int documentsPerId, int batchSize, int fullPushes, int partialPushes) + { + var changeCount = 30; + var expectedPushes = fullPushes + partialPushes; + Config.AzureSearchBatchSize = 10; + + IndexActions = new IndexActions( + new List>( + Enumerable + .Range(0, documentsPerId) + .Select(x => IndexAction.Merge(new KeyedDocument()))), + new List>(), + new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + new Mock().Object)); + + AddChanges(changeCount); + + await Target.ExecuteAsync(); + + VerifyCompletedTelemetry(JobOutcome.Success); + VerifyAllIdsAreProcessed(changeCount); + IndexActionBuilder.Verify( + x => x.UpdateAsync( + It.IsAny(), + It.IsAny>()), + Times.Exactly(changeCount)); + BatchPusher.Verify( + x => x.EnqueueIndexActions(It.IsAny(), It.IsAny()), + Times.Exactly(changeCount)); + BatchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(expectedPushes)); + BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never); + SystemTime.Verify(x => x.Delay(It.IsAny()), Times.Exactly(expectedPushes - 1)); + DownloadDataClient.Verify( + x => x.ReplaceLatestIndexedAsync( + NewDownloadData, + It.Is(a => a.IfMatchETag == OldDownloadResult.Metadata.ETag)), + Times.Once); + + Assert.Equal( + fullPushes, + FinishedBatches.Count(b => b.Sum(ia => ia.Search.Count) == batchSize)); + Assert.Equal( + partialPushes, + FinishedBatches.Count(b => b.Sum(ia => ia.Search.Count) != batchSize)); + Assert.Empty(CurrentBatch); + } + + [Fact] + public async Task CanProcessInParallel() + { + var changeCount = 1000; + Config.AzureSearchBatchSize = 5; + Config.MaxConcurrentBatches = 4; + Config.MaxConcurrentVersionListWriters = 8; + AddChanges(changeCount); + + await Target.ExecuteAsync(); + + VerifyCompletedTelemetry(JobOutcome.Success); + VerifyAllIdsAreProcessed(changeCount); + IndexActionBuilder.Verify( + x => x.UpdateAsync( + It.IsAny(), + It.IsAny>()), + Times.Exactly(changeCount)); + BatchPusher.Verify( + x => x.EnqueueIndexActions(It.IsAny(), It.IsAny()), + Times.Exactly(changeCount)); + BatchPusher.Verify(x => x.TryFinishAsync(), Times.AtLeastOnce); + BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never); + } + + [Fact] + public async Task RetriesFailedPackageIds() + { + Config.MaxConcurrentVersionListWriters = 1; + Changes["PackageA"] = 1; + Changes["PackageB"] = 2; + BatchPusher + .SetupSequence(x => x.TryFinishAsync()) + .ReturnsAsync(new BatchPusherResult(new[] { "PackageB" })) + .ReturnsAsync(new BatchPusherResult()); + + await Target.ExecuteAsync(); + + VerifyCompletedTelemetry(JobOutcome.Success); + VerifyAllIdsAreProcessed(new[] { "PackageA", "PackageB", "PackageB" }); + IndexActionBuilder.Verify( + x => x.UpdateAsync( + "PackageA", + It.IsAny>()), + Times.Once); + IndexActionBuilder.Verify( + x => x.UpdateAsync( + "PackageB", + It.IsAny>()), + Times.Exactly(2)); + BatchPusher.Verify( + x => x.EnqueueIndexActions(It.IsAny(), It.IsAny()), + Times.Exactly(3)); + BatchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(2)); + BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never); + } + + [Fact] + public async Task EventuallyFailsIfBatchPusherNeverSucceeds() + { + Config.MaxConcurrentVersionListWriters = 1; + Changes["PackageA"] = 1; + Changes["PackageB"] = 2; + BatchPusher + .Setup(x => x.TryFinishAsync()) + .ReturnsAsync(new BatchPusherResult(new[] { "PackageB" })); + + var ex = await Assert.ThrowsAsync(() => Target.ExecuteAsync()); + + Assert.Equal("The index operations for the following package IDs failed due to version list concurrency: PackageB", ex.Message); + VerifyCompletedTelemetry(JobOutcome.Failure); + VerifyAllIdsAreProcessed(new[] { "PackageA", "PackageB", "PackageB", "PackageB" }); + IndexActionBuilder.Verify( + x => x.UpdateAsync( + "PackageA", + It.IsAny>()), + Times.Once); + IndexActionBuilder.Verify( + x => x.UpdateAsync( + "PackageB", + It.IsAny>()), + Times.Exactly(3)); + BatchPusher.Verify( + x => x.EnqueueIndexActions(It.IsAny(), It.IsAny()), + Times.Exactly(4)); + BatchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(3)); + BatchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never); + } + + [Fact] + public async Task FailureIsRecordedInTelemetry() + { + var expected = new InvalidOperationException("Something bad!"); + DownloadDataClient + .Setup(x => x.ReadLatestIndexedAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(expected); + + var actual = await Assert.ThrowsAsync(() => Target.ExecuteAsync()); + + VerifyCompletedTelemetry(JobOutcome.Failure); + Assert.Same(expected, actual); + } + + [Theory] + [InlineData(nameof(NewDownloadData))] + [InlineData(nameof(OldDownloadData))] + public async Task RejectsInvalidDataAndNormalizesVersions(string propertyName) + { + var downloadData = (DownloadData)GetType().GetProperty(propertyName).GetValue(this); + downloadData.SetDownloadCount("ValidId", "1.0.0-ValidVersion", 3); + downloadData.SetDownloadCount("ValidId", "1.0.0.a-invalidversion", 5); + downloadData.SetDownloadCount("ValidId", "1.0.0.0-NonNormalized", 7); + downloadData.SetDownloadCount("Invalid--Id", "1.0.0-validversion", 11); + downloadData.SetDownloadCount("Invalid--Id", "1.0.0.a-invalidversion", 13); + + await Target.ExecuteAsync(); + + Assert.Equal(new[] { "ValidId" }, downloadData.Keys.ToArray()); + Assert.Equal(new[] { "1.0.0-NonNormalized", "1.0.0-ValidVersion" }, downloadData["ValidId"].Keys.OrderBy(x => x).ToArray()); + Assert.Equal(10, downloadData.GetDownloadCount("ValidId")); + Assert.Contains("There were 1 invalid IDs, 2 invalid versions, and 1 non-normalized IDs.", Logger.Messages); + } + + [Fact] + public async Task AppliesTransferChanges() + { + var downloadChanges = new SortedDictionary(StringComparer.OrdinalIgnoreCase); + DownloadSetComparer + .Setup(c => c.Compare(It.IsAny(), It.IsAny())) + .Returns((oldData, newData) => + { + return downloadChanges; + }); + + TransferChanges["Package1"] = 100; + TransferChanges["Package2"] = 200; + + NewTransfers.AddTransfer("Package1", "Package2"); + + await Target.ExecuteAsync(); + + PopularityTransferDataClient + .Verify( + c => c.ReadLatestIndexedAsync( + It.Is(x => x.IfMatchETag == null && x.IfNoneMatchETag == null), + It.IsAny()), + Times.Once); + DatabaseFetcher + .Verify( + d => d.GetPopularityTransfersAsync(), + Times.Once); + + DownloadTransferrer + .Verify( + x => x.UpdateDownloadTransfers( + NewDownloadData, + downloadChanges, + OldTransfers, + NewTransfers), + Times.Once); + + // Documents should be updated. + SearchDocumentBuilder + .Verify( + b => b.UpdateDownloadCount("Package1", SearchFilters.IncludePrereleaseAndSemVer2, 100), + Times.Once); + SearchDocumentBuilder + .Verify( + b => b.UpdateDownloadCount("Package2", SearchFilters.IncludePrereleaseAndSemVer2, 200), + Times.Once); + + // Downloads auxiliary file should not include transfer changes. + DownloadDataClient.Verify( + c => c.ReplaceLatestIndexedAsync( + It.Is(d => d.Count == 0), + It.IsAny()), + Times.Once); + + // Popularity transfers auxiliary file should have new data. + PopularityTransferDataClient.Verify( + c => c.ReplaceLatestIndexedAsync( + It.Is(d => + d.Count == 1 && + d["Package1"].Count == 1 && + d["Package1"].Contains("Package2")), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task ConfigDisablesPopularityTransfers() + { + var downloadChanges = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Package1", 5 } + }; + + DownloadSetComparer + .Setup(c => c.Compare(It.IsAny(), It.IsAny())) + .Returns((oldData, newData) => + { + return downloadChanges; + }); + + OldTransfers.AddTransfer("Package1", "Package2"); + NewTransfers.AddTransfer("Package1", "Package2"); + + Config.EnablePopularityTransfers = false; + + await Target.ExecuteAsync(); + + PopularityTransferDataClient + .Verify( + c => c.ReadLatestIndexedAsync(It.IsAny(), It.IsAny()), + Times.Once); + DatabaseFetcher + .Verify( + d => d.GetPopularityTransfersAsync(), + Times.Never); + + // The popularity transfers should not be given to the download transferrer. + DownloadTransferrer + .Verify( + x => x.UpdateDownloadTransfers( + NewDownloadData, + downloadChanges, + OldTransfers, + It.Is(d => d.Count == 0)), + Times.Once); + + // Popularity transfers auxiliary file should be empty. + PopularityTransferDataClient.Verify( + c => c.ReplaceLatestIndexedAsync( + It.Is(d => d.Count == 0), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task FlagDisablesPopularityTransfers() + { + var downloadChanges = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Package1", 5 } + }; + + DownloadSetComparer + .Setup(c => c.Compare(It.IsAny(), It.IsAny())) + .Returns((oldData, newData) => + { + return downloadChanges; + }); + + OldTransfers.AddTransfer("Package1", "Package2"); + NewTransfers.AddTransfer("Package1", "Package2"); + + FeatureFlags + .Setup(x => x.IsPopularityTransferEnabled()) + .Returns(false); + + await Target.ExecuteAsync(); + + PopularityTransferDataClient + .Verify( + c => c.ReadLatestIndexedAsync(It.IsAny(), It.IsAny()), + Times.Once); + DatabaseFetcher + .Verify( + d => d.GetPopularityTransfersAsync(), + Times.Never); + + // The popularity transfers should not be given to the download transferrer. + DownloadTransferrer + .Verify( + x => x.UpdateDownloadTransfers( + NewDownloadData, + downloadChanges, + OldTransfers, + It.Is(d => d.Count == 0)), + Times.Once); + + // Popularity transfers auxiliary file should be empty. + PopularityTransferDataClient.Verify( + c => c.ReplaceLatestIndexedAsync( + It.Is(d => d.Count == 0), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task TransferChangesOverrideDownloadChanges() + { + DownloadSetComparer + .Setup(c => c.Compare(It.IsAny(), It.IsAny())) + .Returns((oldData, newData) => + { + return new SortedDictionary( + newData.ToDictionary(d => d.Key, d => d.Value.Total), + StringComparer.OrdinalIgnoreCase); + }); + + NewDownloadData.SetDownloadCount("A", "1.0.0", 12); + NewDownloadData.SetDownloadCount("A", "2.0.0", 34); + + NewDownloadData.SetDownloadCount("B", "3.0.0", 5); + NewDownloadData.SetDownloadCount("B", "4.0.0", 4); + + NewDownloadData.SetDownloadCount("C", "5.0.0", 2); + NewDownloadData.SetDownloadCount("C", "6.0.0", 3); + + TransferChanges["A"] = 55; + TransferChanges["b"] = 66; + + NewTransfers.AddTransfer("A", "b"); + + await Target.ExecuteAsync(); + + // Documents should have new data with transfer changes. + SearchDocumentBuilder + .Verify( + b => b.UpdateDownloadCount("A", SearchFilters.IncludePrereleaseAndSemVer2, 55), + Times.Once); + SearchDocumentBuilder + .Verify( + b => b.UpdateDownloadCount("B", SearchFilters.IncludePrereleaseAndSemVer2, 66), + Times.Once); + SearchDocumentBuilder + .Verify( + b => b.UpdateDownloadCount("C", SearchFilters.IncludePrereleaseAndSemVer2, 5), + Times.Once); + + // Downloads auxiliary file should not reflect transfer changes. + DownloadDataClient.Verify( + c => c.ReplaceLatestIndexedAsync( + It.Is(d => + d["A"].Total == 46 && + d["A"]["1.0.0"] == 12 && + d["A"]["2.0.0"] == 34 && + + d["B"].Total == 9 && + d["B"]["3.0.0"] == 5 && + d["B"]["4.0.0"] == 4 && + + d["C"].Total == 5 && + d["C"]["5.0.0"] == 2 && + d["C"]["6.0.0"] == 3), + It.IsAny()), + Times.Once); + + // Popularity transfers auxiliary file should have new data. + PopularityTransferDataClient.Verify( + c => c.ReplaceLatestIndexedAsync( + It.Is(d => + d.Count == 1 && + d["A"].Count == 1 && + d["A"].Contains("b")), + It.IsAny()), + Times.Once); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + AuxiliaryFileClient = new Mock(); + DatabaseFetcher = new Mock(); + DownloadDataClient = new Mock(); + DownloadSetComparer = new Mock(); + DownloadTransferrer = new Mock(); + PopularityTransferDataClient = new Mock(); + SearchDocumentBuilder = new Mock(); + IndexActionBuilder = new Mock(); + BatchPusher = new Mock(); + SystemTime = new Mock(); + FeatureFlags = new Mock(); + Options = new Mock>(); + TelemetryService = new Mock(); + Logger = output.GetLogger(); + + Config = new Auxiliary2AzureSearchConfiguration + { + AzureSearchBatchSize = 10, + MaxConcurrentBatches = 1, + MaxConcurrentVersionListWriters = 1, + EnablePopularityTransfers = true, + MinPushPeriod = TimeSpan.FromSeconds(5), + }; + Options.Setup(x => x.Value).Returns(() => Config); + + OldDownloadData = new DownloadData(); + OldDownloadResult = Data.GetAuxiliaryFileResult(OldDownloadData, "download-data-etag"); + DownloadDataClient + .Setup(x => x.ReadLatestIndexedAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => OldDownloadResult); + NewDownloadData = new DownloadData(); + AuxiliaryFileClient.Setup(x => x.LoadDownloadDataAsync()).ReturnsAsync(() => NewDownloadData); + + Changes = new SortedDictionary(); + DownloadSetComparer + .Setup(x => x.Compare(It.IsAny(), It.IsAny())) + .Returns(() => Changes); + + OldTransfers = new PopularityTransferData(); + OldTransferResult = new AuxiliaryFileResult( + modified: true, + data: OldTransfers, + metadata: new AuxiliaryFileMetadata( + DateTimeOffset.UtcNow, + TimeSpan.Zero, + fileSize: 0, + etag: "etag")); + PopularityTransferDataClient + .Setup(x => x.ReadLatestIndexedAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(OldTransferResult); + + NewTransfers = new PopularityTransferData(); + DatabaseFetcher + .Setup(x => x.GetPopularityTransfersAsync()) + .ReturnsAsync(NewTransfers); + + TransferChanges = new SortedDictionary(StringComparer.OrdinalIgnoreCase); + DownloadTransferrer + .Setup(x => x.UpdateDownloadTransfers( + It.IsAny(), + It.IsAny>(), + It.IsAny(), + It.IsAny())) + .Returns(TransferChanges); + + IndexActions = new IndexActions( + new List> { IndexAction.Merge(new KeyedDocument()) }, + new List>(), + new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + Mock.Of())); + ProcessedIds = new ConcurrentBag(); + IndexActionBuilder + .Setup(x => x.UpdateAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => IndexActions) + .Callback>((id, b) => + { + ProcessedIds.Add(id); + b(SearchFilters.IncludePrereleaseAndSemVer2); + }); + + // When pushing, delay for a little bit of time so the stopwatch has some measurable duration. + PushedIds = new ConcurrentBag(); + CurrentBatch = new ConcurrentBag(); + FinishedBatches = new ConcurrentBag>(); + BatchPusher + .Setup(x => x.EnqueueIndexActions(It.IsAny(), It.IsAny())) + .Callback((id, indexActions) => + { + CurrentBatch.Add(indexActions); + PushedIds.Add(id); + }); + BatchPusher + .Setup(x => x.TryFinishAsync()) + .Returns(async () => + { + await Task.Delay(TimeSpan.FromMilliseconds(1)); + return new BatchPusherResult(); + }) + .Callback(() => + { + FinishedBatches.Add(CurrentBatch.ToList()); + CurrentBatch = new ConcurrentBag(); + }); + + FeatureFlags.Setup(x => x.IsPopularityTransferEnabled()).Returns(true); + + Target = new UpdateDownloadsCommand( + AuxiliaryFileClient.Object, + DatabaseFetcher.Object, + DownloadDataClient.Object, + DownloadSetComparer.Object, + DownloadTransferrer.Object, + PopularityTransferDataClient.Object, + SearchDocumentBuilder.Object, + IndexActionBuilder.Object, + () => BatchPusher.Object, + SystemTime.Object, + FeatureFlags.Object, + Options.Object, + TelemetryService.Object, + Logger); + } + + public Mock AuxiliaryFileClient { get; } + public Mock DatabaseFetcher { get; } + public Mock DownloadDataClient { get; } + public Mock DownloadSetComparer { get; } + public Mock DownloadTransferrer { get; } + public Mock PopularityTransferDataClient { get; } + public Mock SearchDocumentBuilder { get; } + public Mock IndexActionBuilder { get; } + public Mock BatchPusher { get; } + public Mock SystemTime { get; } + public Mock FeatureFlags { get; } + public Mock> Options { get; } + public Mock TelemetryService { get; } + public RecordingLogger Logger { get; } + public Auxiliary2AzureSearchConfiguration Config { get; } + public DownloadData OldDownloadData { get; } + public AuxiliaryFileResult OldDownloadResult { get; } + public DownloadData NewDownloadData { get; } + public PopularityTransferData OldTransfers { get; } + public AuxiliaryFileResult OldTransferResult { get; } + public PopularityTransferData NewTransfers { get; } + public SortedDictionary Changes { get; } + public SortedDictionary TransferChanges { get; } + public UpdateDownloadsCommand Target { get; } + public IndexActions IndexActions { get; set; } + public ConcurrentBag ProcessedIds { get; } + public ConcurrentBag PushedIds { get; } + public ConcurrentBag CurrentBatch { get; set; } + public ConcurrentBag> FinishedBatches { get; } + + public void VerifyCompletedTelemetry(JobOutcome outcome) + { + TelemetryService.Verify( + x => x.TrackUpdateDownloadsCompleted(It.IsAny(), It.IsAny()), + Times.Once); + TelemetryService.Verify( + x => x.TrackUpdateDownloadsCompleted(outcome, It.IsAny()), + Times.Once); + } + + public void AddChanges(int changeCount) + { + for (var i = 1; i <= changeCount; i++) + { + Changes[$"Package{i}"] = i; + } + } + + public void VerifyAllIdsAreProcessed(int changeCount) + { + var changedIds = Changes.Keys.OrderBy(x => x).ToArray(); + Assert.Equal(changeCount, changedIds.Length); + VerifyAllIdsAreProcessed(changedIds); + } + + public void VerifyAllIdsAreProcessed(string[] changedIds) + { + var processedIds = ProcessedIds.OrderBy(x => x).ToArray(); + var pushedIds = PushedIds.OrderBy(x => x).ToArray(); + + Assert.Equal(changedIds, processedIds); + Assert.Equal(changedIds, pushedIds); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/UpdateOwnersCommandFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/UpdateOwnersCommandFacts.cs new file mode 100644 index 000000000..42ee5216c --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/UpdateOwnersCommandFacts.cs @@ -0,0 +1,308 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch +{ + public class UpdateOwnersCommandFacts + { + public class ExecuteAsync : Facts + { + public ExecuteAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task DoesNotPushWhenThereAreNoChanges() + { + await Target.ExecuteAsync(); + + Pusher.Verify( + x => x.EnqueueIndexActions(It.IsAny(), It.IsAny()), + Times.Never); + Pusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Never); + Pusher.Verify(x => x.TryFinishAsync(), Times.Never); + OwnerDataClient.Verify(x => x.UploadChangeHistoryAsync(It.IsAny>()), Times.Never); + OwnerDataClient.Verify( + x => x.ReplaceLatestIndexedAsync( + It.IsAny>>(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task ComparesInTheRightOrder() + { + await Target.ExecuteAsync(); + + OwnerSetComparer.Verify( + x => x.CompareOwners( + It.IsAny>>(), + It.IsAny>>()), + Times.Once); + OwnerSetComparer.Verify( + x => x.CompareOwners(StorageResult.Result, DatabaseResult), + Times.Once); + } + + [Fact] + public async Task PushesAllChangesWithSingleWorker() + { + Changes["NuGet.Core"] = new string[0]; + Changes["NuGet.Versioning"] = new string[0]; + Changes["EntityFramework"] = new string[0]; + + await Target.ExecuteAsync(); + + Pusher.Verify( + x => x.EnqueueIndexActions(It.IsAny(), It.IsAny()), + Times.Exactly(3)); + Pusher.Verify( + x => x.EnqueueIndexActions("NuGet.Core", It.IsAny()), + Times.Once); + Pusher.Verify( + x => x.EnqueueIndexActions("NuGet.Versioning", It.IsAny()), + Times.Once); + Pusher.Verify( + x => x.EnqueueIndexActions("EntityFramework", It.IsAny()), + Times.Once); + Pusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Exactly(3)); + Pusher.Verify(x => x.TryFinishAsync(), Times.Once); + } + + [Fact] + public async Task RetriesFailedPushes() + { + Changes["NuGet.Core"] = new string[0]; + Changes["NuGet.Versioning"] = new string[0]; + Changes["EntityFramework"] = new string[0]; + Pusher + .SetupSequence(x => x.TryPushFullBatchesAsync()) + // Attempt #1 + .ReturnsAsync(new BatchPusherResult(new[] { "EntityFramework" })) + .ReturnsAsync(new BatchPusherResult(new[] { "NuGet.Core" })) + .ReturnsAsync(new BatchPusherResult()) + // Attempt #2 + .ReturnsAsync(new BatchPusherResult()) + .ReturnsAsync(new BatchPusherResult()) + .ReturnsAsync(new BatchPusherResult()); + + await Target.ExecuteAsync(); + + Pusher.Verify( + x => x.EnqueueIndexActions(It.IsAny(), It.IsAny()), + Times.Exactly(6)); + Pusher.Verify( + x => x.EnqueueIndexActions("NuGet.Core", It.IsAny()), + Times.Exactly(2)); + Pusher.Verify( + x => x.EnqueueIndexActions("NuGet.Versioning", It.IsAny()), + Times.Exactly(2)); + Pusher.Verify( + x => x.EnqueueIndexActions("EntityFramework", It.IsAny()), + Times.Exactly(2)); + Pusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Exactly(6)); + Pusher.Verify(x => x.TryFinishAsync(), Times.Exactly(2)); + } + + [Fact] + public async Task FailsAfterRetries() + { + Changes["NuGet.Core"] = new string[0]; + Changes["NuGet.Versioning"] = new string[0]; + Changes["EntityFramework"] = new string[0]; + Pusher + .Setup(x => x.TryPushFullBatchesAsync()) + .ReturnsAsync(new BatchPusherResult(new[] { "EntityFramework" })); + + var ex = await Assert.ThrowsAsync(() => Target.ExecuteAsync()); + + Assert.Equal("The index operations for the following package IDs failed due to version list concurrency: EntityFramework", ex.Message); + Pusher.Verify( + x => x.EnqueueIndexActions(It.IsAny(), It.IsAny()), + Times.Exactly(9)); + Pusher.Verify( + x => x.EnqueueIndexActions("NuGet.Core", It.IsAny()), + Times.Exactly(3)); + Pusher.Verify( + x => x.EnqueueIndexActions("NuGet.Versioning", It.IsAny()), + Times.Exactly(3)); + Pusher.Verify( + x => x.EnqueueIndexActions("EntityFramework", It.IsAny()), + Times.Exactly(3)); + Pusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Exactly(9)); + Pusher.Verify(x => x.TryFinishAsync(), Times.Exactly(3)); + } + + [Fact] + public async Task PushesAllChangesWithMultipleWorkers() + { + Configuration.MaxConcurrentBatches = 32; + + Changes["NuGet.Core"] = new string[0]; + Changes["NuGet.Versioning"] = new string[0]; + Changes["EntityFramework"] = new string[0]; + Changes["Microsoft.Extensions.Logging"] = new string[0]; + Changes["Microsoft.Extensions.DependencyInjection"] = new string[0]; + + await Target.ExecuteAsync(); + + Pusher.Verify( + x => x.EnqueueIndexActions(It.IsAny(), It.IsAny()), + Times.Exactly(5)); + Pusher.Verify( + x => x.EnqueueIndexActions("NuGet.Core", It.IsAny()), + Times.Once); + Pusher.Verify( + x => x.EnqueueIndexActions("NuGet.Versioning", It.IsAny()), + Times.Once); + Pusher.Verify( + x => x.EnqueueIndexActions("EntityFramework", It.IsAny()), + Times.Once); + Pusher.Verify( + x => x.EnqueueIndexActions("Microsoft.Extensions.Logging", It.IsAny()), + Times.Once); + Pusher.Verify( + x => x.EnqueueIndexActions("Microsoft.Extensions.DependencyInjection", It.IsAny()), + Times.Once); + Pusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Exactly(5)); + Pusher.Verify(x => x.TryFinishAsync(), Times.Exactly(32)); + } + + [Fact] + public async Task UpdatesBlobStorageAfterIndexing() + { + var actions = new List(); + Pusher + .Setup(x => x.TryFinishAsync()) + .ReturnsAsync(new BatchPusherResult()) + .Callback(() => actions.Add(nameof(IBatchPusher.TryFinishAsync))); + OwnerDataClient + .Setup(x => x.UploadChangeHistoryAsync(It.IsAny>())) + .Returns(Task.CompletedTask) + .Callback(() => actions.Add(nameof(IOwnerDataClient.UploadChangeHistoryAsync))); + OwnerDataClient + .Setup(x => x.ReplaceLatestIndexedAsync(It.IsAny>>(), It.IsAny())) + .Returns(Task.CompletedTask) + .Callback(() => actions.Add(nameof(IOwnerDataClient.ReplaceLatestIndexedAsync))); + + Changes["NuGet.Core"] = new string[0]; + + await Target.ExecuteAsync(); + + Assert.Equal( + new[] { nameof(IBatchPusher.TryFinishAsync), nameof(IOwnerDataClient.UploadChangeHistoryAsync), nameof(IOwnerDataClient.ReplaceLatestIndexedAsync) }, + actions.ToArray()); + } + + [Fact] + public async Task UpdatesBlobStorage() + { + IReadOnlyList changeHistory = null; + OwnerDataClient + .Setup(x => x.UploadChangeHistoryAsync(It.IsAny>())) + .Returns(Task.CompletedTask) + .Callback>(x => changeHistory = x); + + Changes["NuGet.Versioning"] = new string[0]; + Changes["NuGet.Core"] = new string[0]; + + await Target.ExecuteAsync(); + + Assert.Equal(new[] { "NuGet.Core", "NuGet.Versioning" }, changeHistory.ToArray()); + OwnerDataClient.Verify( + x => x.ReplaceLatestIndexedAsync(DatabaseResult, StorageResult.AccessCondition), + Times.Once); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + DatabaseOwnerFetcher = new Mock(); + OwnerDataClient = new Mock(); + OwnerSetComparer = new Mock(); + SearchDocumentBuilder = new Mock(); + SearchIndexActionBuilder = new Mock(); + Pusher = new Mock(); + Options = new Mock>(); + TelemetryService = new Mock(); + Logger = output.GetLogger(); + + Configuration = new AzureSearchJobConfiguration + { + MaxConcurrentBatches = 1, + }; + DatabaseResult = new SortedDictionary>(); + StorageResult = new ResultAndAccessCondition>>( + new SortedDictionary>(), + new Mock().Object); + Changes = new SortedDictionary(); + IndexActions = new IndexActions( + new List> { IndexAction.Merge(new KeyedDocument()) }, + new List> { IndexAction.Merge(new KeyedDocument()) }, + new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + new Mock().Object)); + + Pusher.SetReturnsDefault(Task.FromResult(new BatchPusherResult())); + Options + .Setup(x => x.Value) + .Returns(() => Configuration); + DatabaseOwnerFetcher + .Setup(x => x.GetPackageIdToOwnersAsync()) + .ReturnsAsync(() => DatabaseResult); + OwnerDataClient + .Setup(x => x.ReadLatestIndexedAsync()) + .ReturnsAsync(() => StorageResult); + OwnerSetComparer + .Setup(x => x.CompareOwners( + It.IsAny>>(), + It.IsAny>>())) + .Returns(() => Changes); + SearchIndexActionBuilder + .Setup(x => x.UpdateAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => IndexActions); + + Target = new UpdateOwnersCommand( + DatabaseOwnerFetcher.Object, + OwnerDataClient.Object, + OwnerSetComparer.Object, + SearchDocumentBuilder.Object, + SearchIndexActionBuilder.Object, + () => Pusher.Object, + Options.Object, + TelemetryService.Object, + Logger); + } + + public Mock DatabaseOwnerFetcher { get; } + public Mock OwnerDataClient { get; } + public Mock OwnerSetComparer { get; } + public Mock SearchDocumentBuilder { get; } + public Mock SearchIndexActionBuilder { get; } + public Mock Pusher { get; } + public Mock> Options { get; } + public Mock TelemetryService { get; } + public RecordingLogger Logger { get; } + public AzureSearchJobConfiguration Configuration { get; } + public SortedDictionary> DatabaseResult { get; } + public ResultAndAccessCondition>> StorageResult { get; } + public SortedDictionary Changes { get; } + public IndexActions IndexActions { get; } + public UpdateOwnersCommand Target { get; } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/UpdateVerifiedPackagesCommandFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/UpdateVerifiedPackagesCommandFacts.cs new file mode 100644 index 000000000..a34d3aede --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Auxiliary2AzureSearch/UpdateVerifiedPackagesCommandFacts.cs @@ -0,0 +1,110 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Moq; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.AzureSearch.Support; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Auxiliary2AzureSearch +{ + public class UpdateVerifiedPackagesCommandFacts + { + public class ExecuteAsync : Facts + { + public ExecuteAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task PushesAddedVerifiedPackage() + { + NewVerifiedPackagesData.Add("NuGet.Versioning"); + + await Target.ExecuteAsync(); + + VerifyCompletedTelemetry(JobOutcome.Success); + VerifiedPackagesDataClient.Verify( + x => x.ReplaceLatestAsync( + NewVerifiedPackagesData, + It.Is(a => a.IfMatchETag == OldVerifiedPackagesResult.Metadata.ETag)), + Times.Once); + } + + [Fact] + public async Task PushesRemovedVerifiedPackage() + { + OldVerifiedPackagesData.Add("NuGet.Versioning"); + + await Target.ExecuteAsync(); + + VerifyCompletedTelemetry(JobOutcome.Success); + VerifiedPackagesDataClient.Verify( + x => x.ReplaceLatestAsync( + NewVerifiedPackagesData, + It.Is(a => a.IfMatchETag == OldVerifiedPackagesResult.Metadata.ETag)), + Times.Once); + } + + [Fact] + public async Task DoesNotPushUnchangedVerifiedPackages() + { + await Target.ExecuteAsync(); + + VerifyCompletedTelemetry(JobOutcome.NoOp); + VerifiedPackagesDataClient.Verify( + x => x.ReplaceLatestAsync(It.IsAny>(), It.IsAny()), + Times.Never); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + DatabaseAuxiliaryDataFetcher = new Mock(); + VerifiedPackagesDataClient = new Mock(); + TelemetryService = new Mock(); + Logger = output.GetLogger(); + + OldVerifiedPackagesData = new HashSet(); + OldVerifiedPackagesResult = Data.GetAuxiliaryFileResult(OldVerifiedPackagesData, "verified-packages-etag"); + VerifiedPackagesDataClient + .Setup(x => x.ReadLatestAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => OldVerifiedPackagesResult); + NewVerifiedPackagesData = new HashSet(); + DatabaseAuxiliaryDataFetcher.Setup(x => x.GetVerifiedPackagesAsync()).ReturnsAsync(() => NewVerifiedPackagesData); + + Target = new UpdateVerifiedPackagesCommand( + DatabaseAuxiliaryDataFetcher.Object, + VerifiedPackagesDataClient.Object, + TelemetryService.Object, + Logger); + } + + public Mock DatabaseAuxiliaryDataFetcher { get; } + public Mock VerifiedPackagesDataClient { get; } + public Mock TelemetryService { get; } + public RecordingLogger Logger { get; } + public UpdateVerifiedPackagesCommand Target { get; } + public HashSet OldVerifiedPackagesData { get; } + public AuxiliaryFileResult> OldVerifiedPackagesResult { get; } + public HashSet NewVerifiedPackagesData { get; } + + public void VerifyCompletedTelemetry(JobOutcome outcome) + { + TelemetryService.Verify( + x => x.TrackUpdateVerifiedPackagesCompleted(It.IsAny(), It.IsAny()), + Times.Once); + TelemetryService.Verify( + x => x.TrackUpdateVerifiedPackagesCompleted(outcome, It.IsAny()), + Times.Once); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/AuxiliaryFileClientFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/AuxiliaryFileClientFacts.cs new file mode 100644 index 000000000..0b5ee6079 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/AuxiliaryFileClientFacts.cs @@ -0,0 +1,177 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.IO; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Moq; +using NuGet.Services.AzureSearch.Db2AzureSearch; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class AuxiliaryFileClientFacts + { + public class LoadDownloadDataAsync : BaseFacts + { + public LoadDownloadDataAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task ReadsContent() + { + var json = @" +[ + [ + ""NuGet.Frameworks"", + [ ""1.0.0"", 406], + [ ""2.0.0-ALPHA"", 137] + ], + [ + ""NuGet.Versioning"", + [""3.0.0"", 138] + ] +] +"; + _blob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var actual = await _target.LoadDownloadDataAsync(); + + Assert.NotNull(actual); + Assert.Equal(406, actual["NuGet.Frameworks"]["1.0.0"]); + Assert.Equal(137, actual["NuGet.Frameworks"]["2.0.0-alpha"]); + Assert.Equal(138, actual["nuget.versioning"]["3.0.0"]); + Assert.Equal(0, actual["nuget.versioning"].GetDownloadCount("4.0.0")); + Assert.Equal(0, actual.GetDownloadCount("something.else")); + _blobClient.Verify(x => x.GetContainerReference("my-container"), Times.Once); + _blobClient.Verify(x => x.GetContainerReference(It.IsAny()), Times.Once); + _container.Verify(x => x.GetBlobReference("my-downloads.json"), Times.Once); + _container.Verify(x => x.GetBlobReference(It.IsAny()), Times.Once); + _blob.Verify(x => x.OpenReadAsync(It.Is(a => a.IfMatchETag == null && a.IfNoneMatchETag == null)), Times.Once); + _blob.Verify(x => x.OpenReadAsync(It.IsAny()), Times.Once); + } + } + + public class LoadExcludedPackagesAsync : BaseFacts + { + public LoadExcludedPackagesAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task ReadsContent() + { + var json = @" +[ + ""NuGet.Frameworks"", + ""NuGet.Versioning"" +] +"; + _blob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var actual = await _target.LoadExcludedPackagesAsync(); + + Assert.NotNull(actual); + Assert.Contains("NuGet.Frameworks", actual); + Assert.Contains("nuget.versioning", actual); + Assert.DoesNotContain("something.else", actual); + _blobClient.Verify(x => x.GetContainerReference("my-container"), Times.Once); + _blobClient.Verify(x => x.GetContainerReference(It.IsAny()), Times.Once); + _container.Verify(x => x.GetBlobReference("my-excluded-packages.json"), Times.Once); + _container.Verify(x => x.GetBlobReference(It.IsAny()), Times.Once); + _blob.Verify(x => x.OpenReadAsync(It.Is(a => a.IfMatchETag == null && a.IfNoneMatchETag == null)), Times.Once); + _blob.Verify(x => x.OpenReadAsync(It.IsAny()), Times.Once); + } + + [Fact] + public async Task ThrowsStorageExceptionWhenNotFound() + { + _blob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ThrowsAsync(new StorageException( + res: new RequestResult() + { + HttpStatusCode = (int)HttpStatusCode.NotFound, + }, + message: "Not so fast, buddy!", + inner: null)); + + var exception = await Assert.ThrowsAsync(async () => await _target.LoadExcludedPackagesAsync()); + Assert.True(exception.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.NotFound); + } + } + + public abstract class BaseFacts + { + protected readonly Mock _blobClient; + protected readonly Db2AzureSearchConfiguration _config; + protected readonly AuxiliaryDataStorageConfiguration _configStorage; + protected readonly Mock> _options; + protected readonly Mock> _optionsStorage; + protected readonly Mock _telemetryService; + protected readonly RecordingLogger _logger; + protected readonly Mock _container; + protected readonly Mock _blob; + protected readonly string _etag; + protected readonly AuxiliaryFileClient _target; + + public BaseFacts(ITestOutputHelper output) + { + _blobClient = new Mock(); + _config = new Db2AzureSearchConfiguration(); + _configStorage = new AuxiliaryDataStorageConfiguration(); + _options = new Mock>(); + _optionsStorage = new Mock>(); + _telemetryService = new Mock(); + _logger = output.GetLogger(); + _container = new Mock(); + _blob = new Mock(); + _etag = "\"something\""; + + _config.AuxiliaryDataStorageContainer = "my-container"; + _config.AuxiliaryDataStorageDownloadsPath = "my-downloads.json"; + _config.AuxiliaryDataStorageVerifiedPackagesPath = "my-verified-packages.json"; + _config.AuxiliaryDataStorageExcludedPackagesPath = "my-excluded-packages.json"; + _options.Setup(x => x.Value).Returns(() => _config); + + _configStorage.AuxiliaryDataStorageContainer = "my-container"; + _configStorage.AuxiliaryDataStorageDownloadsPath = "my-downloads.json"; + _configStorage.AuxiliaryDataStorageExcludedPackagesPath = "my-excluded-packages.json"; + _optionsStorage.Setup(x => x.Value).Returns(() => _configStorage); + + _blobClient + .Setup(x => x.GetContainerReference(It.IsAny())) + .Returns(() => _container.Object); + _container + .Setup(x => x.GetBlobReference(It.IsAny())) + .Returns(() => _blob.Object); + _blob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes("[]"))); + _blob + .Setup(x => x.ETag) + .Returns(() => _etag); + _blob + .Setup(x => x.Properties) + .Returns(new BlobProperties()); + + _target = new AuxiliaryFileClient( + _blobClient.Object, + _optionsStorage.Object, + _telemetryService.Object, + _logger); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/DownloadByVersionDataFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/DownloadByVersionDataFacts.cs new file mode 100644 index 000000000..5816c3aaf --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/DownloadByVersionDataFacts.cs @@ -0,0 +1,146 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using Xunit; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class DownloadsByVersionDataFacts + { + public class Total : Facts + { + [Fact] + public void StartsWithZero() + { + Assert.Equal(0, Target.Total); + } + + [Fact] + public void HasAllVersionCounts() + { + Target.SetDownloadCount(V1, 10); + Target.SetDownloadCount(V2, 11); + + Assert.Equal(21, Target.Total); + } + } + + public class GetDownloadCount : Facts + { + [Fact] + public void ReturnsDownloadCount() + { + Target.SetDownloadCount(V1, 10); + + Assert.Equal(10, Target.GetDownloadCount(V1)); + } + + [Fact] + public void AllowsDifferentCase() + { + Target.SetDownloadCount(V1, 10); + + Assert.Equal(10, Target.GetDownloadCount(V1Upper)); + } + + [Fact] + public void ReturnsZeroForMissingVersion() + { + Assert.Equal(0, Target.GetDownloadCount(V1)); + } + } + + public class SetDownloadCount : Facts + { + [Fact] + public void AllowsUpdatingDownloadCount() + { + Target.SetDownloadCount(V1, 10); + Target.SetDownloadCount(V1, 1); + + Assert.Equal(1, Target.GetDownloadCount(V1)); + Assert.Equal(1, Target.Total); + } + + [Fact] + public void AllowsUpdatingDownloadCountWithDifferentCase() + { + Target.SetDownloadCount(V1, 10); + Target.SetDownloadCount(V2, 5); + Target.SetDownloadCount(V1Upper, 1); + + Assert.Equal(1, Target.GetDownloadCount(V1)); + Assert.Equal(6, Target.Total); + } + + [Fact] + public void ReplacesCaseOfVersionString() + { + Target.SetDownloadCount(V1, 10); + Target.SetDownloadCount(V1Upper, 10); + + var pair = Assert.Single(Target); + Assert.Equal(V1Upper, pair.Key); + Assert.Equal(10, pair.Value); + } + + [Fact] + public void RemovesVersionWithZeroDownloads() + { + Target.SetDownloadCount(V1, 10); + Target.SetDownloadCount(V1Upper, 0); + + Assert.Empty(Target); + } + + [Fact] + public void RejectsNegativeDownloadCount() + { + var ex = Assert.Throws(() => Target.SetDownloadCount(V1, -1)); + Assert.Contains("The download count must not be negative.", ex.Message); + Assert.Equal("downloads", ex.ParamName); + } + } + + public class EnumerableImplementation : Facts + { + [Fact] + public void ReturnsAllVersions() + { + Target.SetDownloadCount(V2, 2); + Target.SetDownloadCount(V3, 3); + Target.SetDownloadCount(V0, 0); + Target.SetDownloadCount(V1Upper, 1); + + var items = Target.OrderBy(x => x.Key).ToArray(); + + Assert.Equal( + new[] + { + KeyValuePair.Create(V1Upper, 1L), + KeyValuePair.Create(V2, 2L), + KeyValuePair.Create(V3, 3L), + }, + items); + } + } + + public abstract class Facts + { + public const string V0 = "0.0.0"; + public const string V1 = "1.0.0-alpha"; + public const string V1Upper = "1.0.0-ALPHA"; + public const string V2 = "2.0.0"; + public const string V3 = "3.0.0"; + + public Facts() + { + Target = new DownloadByVersionData(); + } + + public DownloadByVersionData Target { get; } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/DownloadDataClientFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/DownloadDataClientFacts.cs new file mode 100644 index 000000000..a69369681 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/DownloadDataClientFacts.cs @@ -0,0 +1,355 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Moq; +using Newtonsoft.Json; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class DownloadDataClientFacts + { + public class ReadLatestIndexedAsync : Facts + { + public ReadLatestIndexedAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task AllowsEmptyObject() + { + var json = JsonConvert.SerializeObject(new Dictionary()); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestIndexedAsync(AccessCondition.Object, StringCache); + + Assert.True(output.Modified); + Assert.Empty(output.Data); + Assert.Equal(ETag, output.Metadata.ETag); + } + + [Fact] + public async Task AllowsNotModifiedBlob() + { + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ThrowsAsync(new StorageException( + new RequestResult + { + HttpStatusCode = (int)HttpStatusCode.NotModified, + }, + message: "Not modified.", + inner: null)); + + var output = await Target.ReadLatestIndexedAsync(AccessCondition.Object, StringCache); + + Assert.False(output.Modified); + Assert.Null(output.Data); + Assert.Null(output.Metadata); + } + + [Fact] + public async Task RejectsMissingBlob() + { + var expected = new StorageException( + new RequestResult + { + HttpStatusCode = (int)HttpStatusCode.NotFound, + }, + message: "Not found.", + inner: null); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ThrowsAsync(expected); + + var actual = await Assert.ThrowsAsync( + () => Target.ReadLatestIndexedAsync(AccessCondition.Object, StringCache)); + Assert.Same(actual, expected); + } + + [Fact] + public async Task RejectsInvalidJson() + { + var json = JsonConvert.SerializeObject(new object[] + { + new object[] + { + "nuget.versioning", + new object[] + { + new object[] { "1.0.0", 5 }, + }, + }, + new object[] + { + "EntityFramework", + new object[] + { + new object[] { "2.0.0", 10 }, + }, + } + }); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var ex = await Assert.ThrowsAsync( + () => Target.ReadLatestIndexedAsync(AccessCondition.Object, StringCache)); + Assert.Equal("The first token should be the start of an object.", ex.Message); + } + + [Fact] + public async Task DedupesStrings() + { + var json = JsonConvert.SerializeObject(new Dictionary> + { + { + "nuget.versioning", + new Dictionary + { + { "1.0.0", 1 }, + } + }, + { + "EntityFramework", + new Dictionary + { + { "1.0.0", 10 }, + } + }, + }); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestIndexedAsync(AccessCondition.Object, StringCache); + + var versionA = output.Data["nuget.versioning"].Single().Key; + var versionB = output.Data["EntityFramework"].Single().Key; + Assert.Same(versionA, versionB); + } + + [Fact] + public async Task ReadsDownloads() + { + var json = JsonConvert.SerializeObject(new Dictionary> + { + { + "nuget.versioning", + new Dictionary + { + { "1.0.0", 1 }, + { "2.0.0-alpha", 5 }, + } + }, + { + "NuGet.Core", + new Dictionary() + }, + { + "EntityFramework", + new Dictionary + { + { "2.0.0", 10 }, + } + }, + }); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestIndexedAsync(AccessCondition.Object, StringCache); + + Assert.True(output.Modified); + Assert.Equal(new[] { "EntityFramework", "nuget.versioning" }, output.Data.Select(x => x.Key).OrderBy(x => x).ToArray()); + Assert.Equal(6, output.Data.GetDownloadCount("NuGet.Versioning")); + Assert.Equal(1, output.Data.GetDownloadCount("NuGet.Versioning", "1.0.0")); + Assert.Equal(5, output.Data.GetDownloadCount("NuGet.Versioning", "2.0.0-ALPHA")); + Assert.Equal(10, output.Data.GetDownloadCount("EntityFramework")); + Assert.Equal(ETag, output.Metadata.ETag); + + CloudBlobContainer.Verify(x => x.GetBlobReference("downloads/downloads.v2.json"), Times.Once); + } + } + + public class ReplaceLatestIndexedAsync : Facts + { + public ReplaceLatestIndexedAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task SerializesWithoutBOM() + { + var newData = new DownloadData(); + + await Target.ReplaceLatestIndexedAsync(newData, AccessCondition.Object); + + var bytes = Assert.Single(SavedBytes); + Assert.Equal((byte)'{', bytes[0]); + } + + [Fact] + public async Task SetsContentType() + { + var newData = new DownloadData(); + + await Target.ReplaceLatestIndexedAsync(newData, AccessCondition.Object); + + Assert.Equal("application/json", CloudBlob.Object.Properties.ContentType); + } + + [Fact] + public async Task SerializedWithoutIndentation() + { + var newData = new DownloadData(); + newData.SetDownloadCount("nuget.versioning", "1.0.0", 1); + newData.SetDownloadCount("NuGet.Versioning", "2.0.0", 5); + newData.SetDownloadCount("EntityFramework", "3.0.0", 10); + + await Target.ReplaceLatestIndexedAsync(newData, AccessCondition.Object); + + var json = Assert.Single(SavedStrings); + Assert.DoesNotContain("\n", json); + } + + [Fact] + public async Task Serializes() + { + var newData = new DownloadData(); + newData.SetDownloadCount("ZZZ", "9.0.0", 23); + newData.SetDownloadCount("YYY", "9.0.0", 0); + newData.SetDownloadCount("nuget.versioning", "1.0.0", 1); + newData.SetDownloadCount("NuGet.Versioning", "2.0.0", 5); + newData.SetDownloadCount("EntityFramework", "3.0.0", 10); + newData.SetDownloadCount("EntityFramework", "1.0.0", 0); + + await Target.ReplaceLatestIndexedAsync(newData, AccessCondition.Object); + + // Pretty-ify and sort the JSON to make the assertion clearer. + var json = Assert.Single(SavedStrings); + var dictionary = JsonConvert.DeserializeObject>>(json); + json = JsonConvert.SerializeObject(dictionary, Formatting.Indented); + + Assert.Equal(@"{ + ""EntityFramework"": { + ""3.0.0"": 10 + }, + ""NuGet.Versioning"": { + ""1.0.0"": 1, + ""2.0.0"": 5 + }, + ""ZZZ"": { + ""9.0.0"": 23 + } +}", json); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + CloudBlobClient = new Mock(); + CloudBlobContainer = new Mock(); + CloudBlob = new Mock(); + Options = new Mock>(); + TelemetryService = new Mock(); + Logger = output.GetLogger(); + Config = new AzureSearchJobConfiguration + { + StorageContainer = "unit-test-container", + }; + + ETag = "\"some-etag\""; + AccessCondition = new Mock(); + StringCache = new StringCache(); + + Options + .Setup(x => x.Value) + .Returns(() => Config); + CloudBlobClient + .Setup(x => x.GetContainerReference(It.IsAny())) + .Returns(() => CloudBlobContainer.Object); + CloudBlobContainer + .Setup(x => x.GetBlobReference(It.IsAny())) + .Returns(() => CloudBlob.Object) + .Callback(x => BlobNames.Add(x)); + CloudBlob + .Setup(x => x.ETag) + .Returns(ETag); + CloudBlob + .Setup(x => x.OpenWriteAsync(It.IsAny())) + .ReturnsAsync(() => new RecordingStream(bytes => + { + SavedBytes.Add(bytes); + SavedStrings.Add(Encoding.UTF8.GetString(bytes)); + })); + CloudBlob + .Setup(x => x.Properties) + .Returns(new CloudBlockBlob(new Uri("https://example/blob")).Properties); + + Target = new DownloadDataClient( + CloudBlobClient.Object, + Options.Object, + TelemetryService.Object, + Logger); + } + + public Mock CloudBlobClient { get; } + public Mock CloudBlobContainer { get; } + public Mock CloudBlob { get; } + public Mock> Options { get; } + public Mock TelemetryService { get; } + public RecordingLogger Logger { get; } + public AzureSearchJobConfiguration Config { get; } + public string ETag { get; } + public Mock AccessCondition { get; } + public StringCache StringCache { get; } + public DownloadDataClient Target { get; } + + public List BlobNames { get; } = new List(); + public List SavedBytes { get; } = new List(); + public List SavedStrings { get; } = new List(); + } + + private class RecordingStream : MemoryStream + { + private readonly object _lock = new object(); + private Action _onDispose; + + public RecordingStream(Action onDispose) + { + _onDispose = onDispose; + } + + protected override void Dispose(bool disposing) + { + lock (_lock) + { + if (_onDispose != null) + { + _onDispose(ToArray()); + _onDispose = null; + } + } + + base.Dispose(disposing); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/DownloadDataFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/DownloadDataFacts.cs new file mode 100644 index 000000000..2bbc0c99e --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/DownloadDataFacts.cs @@ -0,0 +1,127 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Xunit; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class DownloadDataFacts + { + public class GetDownloadCountById : Facts + { + [Fact] + public void ReturnsZeroForUnknownId() + { + Assert.Equal(0, Target.GetDownloadCount(IdA)); + } + + [Fact] + public void ReturnsTotalForId() + { + Target.SetDownloadCount(IdA, V1, 1); + Target.SetDownloadCount(IdB, V2, 5); + Target.SetDownloadCount(IdA, V3, 10); + + Assert.Equal(11, Target.GetDownloadCount(IdA)); + } + } + + public class GetDownloadCountByIdAndVersion : Facts + { + [Fact] + public void ReturnsZeroForUnknownIdAndVersion() + { + Assert.Equal(0, Target.GetDownloadCount(IdA, V1)); + } + + [Fact] + public void ReturnsZeroForUnknownVersion() + { + Target.SetDownloadCount(IdA, V1, 1); + + Assert.Equal(0, Target.GetDownloadCount(IdA, V2)); + } + + [Fact] + public void ReturnsDownloadsForVersion() + { + Target.SetDownloadCount(IdA, V1, 1); + + Assert.Equal(1, Target.GetDownloadCount(IdA)); + } + } + + public class SetDownloadCount : Facts + { + [Fact] + public void AllowsUpdatingDownloadCount() + { + Target.SetDownloadCount(IdA, V1, 10); + Target.SetDownloadCount(IdA, V1, 1); + + Assert.Equal(1, Target.GetDownloadCount(IdA, V1)); + Assert.Equal(1, Target.GetDownloadCount(IdA)); + } + + [Fact] + public void AllowsUpdatingDownloadCountWithDifferentCase() + { + Target.SetDownloadCount(IdA, V1, 10); + Target.SetDownloadCount(IdA, V2, 5); + Target.SetDownloadCount(IdAUpper, V1, 1); + + Assert.Equal(1, Target.GetDownloadCount(IdA, V1)); + Assert.Equal(6, Target.GetDownloadCount(IdA)); + } + + [Fact] + public void ReplacesCaseOfVersionString() + { + Target.SetDownloadCount(IdA, V1, 10); + Target.SetDownloadCount(IdAUpper, V1, 10); + + var pair = Assert.Single(Target); + Assert.Equal(IdAUpper, pair.Key); + Assert.Equal(10, pair.Value.Total); + } + + [Fact] + public void RemovesVersionWithZeroDownloads() + { + Target.SetDownloadCount(IdA, V1, 10); + Target.SetDownloadCount(IdA, V1Upper, 0); + + Assert.Empty(Target); + } + + [Fact] + public void RejectsNegativeDownloadCount() + { + var ex = Assert.Throws(() => Target.SetDownloadCount(IdA, V1, -1)); + Assert.Contains("The download count must not be negative.", ex.Message); + Assert.Equal("downloads", ex.ParamName); + } + } + + public abstract class Facts + { + public const string V0 = "0.0.0"; + public const string V1 = "1.0.0-alpha"; + public const string V1Upper = "1.0.0-ALPHA"; + public const string V2 = "2.0.0"; + public const string V3 = "3.0.0"; + + public const string IdA = "NuGet.Frameworks"; + public const string IdAUpper = "NUGET.FRAMEWORKS"; + public const string IdB = "NuGet.Versioning"; + + public Facts() + { + Target = new DownloadData(); + } + + public DownloadData Target { get; } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/JsonStringArrayFileParsingTests.cs b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/JsonStringArrayFileParsingTests.cs new file mode 100644 index 000000000..7b8bfb70c --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/JsonStringArrayFileParsingTests.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Newtonsoft.Json; +using Xunit; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class JsonStringArrayFileParserTest + { + [Theory] + [InlineData("[]", new string[0])] + [InlineData("['test']", new string[] { "test" })] + [InlineData("['test', 'test']", new string[] { "test" })] + [InlineData("['test', 'test123']", new string[] { "test", "test123" })] + [InlineData("['test', 'Test123', 'test123', 'tEst123']", new string[] { "test", "test123" })] + public void ParsesProperInput(string input, string[] expected) + { + var actual = Parse(input); + + Assert.Equal(expected.Length, actual.Count); + Assert.True(expected.All(actual.Contains)); + } + + [Theory] + [InlineData("{'test':'hi'}")] + [InlineData("['test', 'test123'")] + [InlineData("['test', {'test':'hi'}]")] + public void ThrowsOnInvalidInput(string input) + { + Assert.ThrowsAny(() => Parse(input)); + } + + private HashSet Parse(string input) + { + return JsonStringArrayFileParser.Parse(new JsonTextReader(new StringReader(input))); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/OwnerDataClientFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/OwnerDataClientFacts.cs new file mode 100644 index 000000000..b091a4f88 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/OwnerDataClientFacts.cs @@ -0,0 +1,488 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Moq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class OwnerDataClientFacts + { + public class ReadLatestIndexedAsync : Facts + { + public ReadLatestIndexedAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task AllowsEmptyObject() + { + var json = JsonConvert.SerializeObject(new Dictionary()); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestIndexedAsync(); + + Assert.Empty(output.Result); + Assert.Equal(ETag, output.AccessCondition.IfMatchETag); + + TelemetryService.Verify( + x => x.TrackReadLatestIndexedOwners( + /* packageIdCount: */ 0, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task AllowsMissingBlob() + { + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ThrowsAsync(new StorageException( + new RequestResult + { + HttpStatusCode = (int)HttpStatusCode.NotFound, + }, + message: "Not found.", + inner: null)); + + var output = await Target.ReadLatestIndexedAsync(); + + Assert.Empty(output.Result); + Assert.Equal("*", output.AccessCondition.IfNoneMatchETag); + + TelemetryService.Verify( + x => x.TrackReadLatestIndexedOwners( + /* packageIdCount: */ 0, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task RejectsInvalidJson() + { + var json = JsonConvert.SerializeObject(new object[] + { + new object[] + { + "nuget.versioning", + new[] { "nuget", "Microsoft" } + }, + new object[] + { + "EntityFramework", + new[] { "Microsoft", "aspnet", "EntityFramework" } + } + }); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var ex = await Assert.ThrowsAsync( + () => Target.ReadLatestIndexedAsync()); + Assert.Equal("The first token should be the start of an object.", ex.Message); + + TelemetryService.Verify( + x => x.TrackReadLatestIndexedOwners( + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task ReadsOwners() + { + var json = JsonConvert.SerializeObject(new Dictionary + { + { + "nuget.versioning", + new[] { "nuget", "Microsoft" } + }, + { + "EntityFramework", + new[] { "Microsoft", "aspnet", "EntityFramework" } + }, + { + "NuGet.Core", + new[] { "nuget" } + }, + { + "ZDuplicate", + new[] { "ownerA", "ownera", "OWNERA", "ownerB" } + }, + }); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestIndexedAsync(); + + Assert.Equal(4, output.Result.Count); + Assert.Equal(new[] { "EntityFramework", "NuGet.Core", "nuget.versioning", "ZDuplicate" }, output.Result.Keys.ToArray()); + Assert.Equal(new[] { "aspnet", "EntityFramework", "Microsoft" }, output.Result["EntityFramework"].ToArray()); + Assert.Equal(new[] { "nuget" }, output.Result["NuGet.Core"].ToArray()); + Assert.Equal(new[] { "Microsoft", "nuget" }, output.Result["nuget.versioning"].ToArray()); + Assert.Equal(new[] { "ownerA", "ownerB" }, output.Result["ZDuplicate"].ToArray()); + Assert.Equal(StringComparer.OrdinalIgnoreCase, output.Result.Comparer); + Assert.Equal(StringComparer.OrdinalIgnoreCase, output.Result["EntityFramework"].Comparer); + Assert.Equal(StringComparer.OrdinalIgnoreCase, output.Result["NuGet.Core"].Comparer); + Assert.Equal(StringComparer.OrdinalIgnoreCase, output.Result["nuget.versioning"].Comparer); + Assert.Equal(StringComparer.OrdinalIgnoreCase, output.Result["ZDuplicate"].Comparer); + Assert.Equal(ETag, output.AccessCondition.IfMatchETag); + + CloudBlobContainer.Verify(x => x.GetBlobReference("owners/owners.v2.json"), Times.Once); + + TelemetryService.Verify( + x => x.TrackReadLatestIndexedOwners( + /* packageIdCount: */ 4, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task IgnoresEmptyOwnerLists() + { + var json = JsonConvert.SerializeObject(new Dictionary + { + { + "NoOwners", + new string[0] + }, + { + "NuGet.Core", + new[] { "nuget" } + }, + }); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestIndexedAsync(); + + Assert.Single(output.Result); + Assert.Equal(new[] { "NuGet.Core" }, output.Result.Keys.ToArray()); + Assert.Equal(new[] { "nuget" }, output.Result["NuGet.Core"].ToArray()); + Assert.Equal(StringComparer.OrdinalIgnoreCase, output.Result.Comparer); + Assert.Equal(StringComparer.OrdinalIgnoreCase, output.Result["NuGet.Core"].Comparer); + Assert.Equal(ETag, output.AccessCondition.IfMatchETag); + + TelemetryService.Verify( + x => x.TrackReadLatestIndexedOwners( + /* packageIdCount: */ 1, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task AllowsDuplicateIds() + { + var json = @" +{ + ""NuGet.Core"": [ ""nuget"" ], + ""NuGet.Core"": [ ""aspnet"" ], + ""nuget.core"": [ ""microsoft"" ] +}"; + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestIndexedAsync(); + + Assert.Single(output.Result); + Assert.Equal(new[] { "NuGet.Core" }, output.Result.Keys.ToArray()); + Assert.Equal(new[] { "aspnet", "microsoft", "nuget" }, output.Result["NuGet.Core"].ToArray()); + Assert.Equal(StringComparer.OrdinalIgnoreCase, output.Result.Comparer); + Assert.Equal(StringComparer.OrdinalIgnoreCase, output.Result["NuGet.Core"].Comparer); + Assert.Equal(ETag, output.AccessCondition.IfMatchETag); + + TelemetryService.Verify( + x => x.TrackReadLatestIndexedOwners( + /* packageIdCount: */ 1, + It.IsAny()), + Times.Once); + } + } + + public class ReplaceLatestIndexedAsync : Facts + { + public ReplaceLatestIndexedAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task SerializesWithoutBOM() + { + var newData = new SortedDictionary>(); + + await Target.ReplaceLatestIndexedAsync(newData, AccessCondition.Object); + + var bytes = Assert.Single(SavedBytes); + Assert.Equal((byte)'{', bytes[0]); + } + + [Fact] + public async Task SetsContentType() + { + var newData = new SortedDictionary>(); + + await Target.ReplaceLatestIndexedAsync(newData, AccessCondition.Object); + + Assert.Equal("application/json", CloudBlob.Object.Properties.ContentType); + } + + [Fact] + public async Task SerializedWithoutIndentation() + { + var newData = new SortedDictionary>(StringComparer.OrdinalIgnoreCase) + { + { + "nuget.versioning", + new SortedSet(StringComparer.OrdinalIgnoreCase) { "nuget", "Microsoft" } + } + }; + + await Target.ReplaceLatestIndexedAsync(newData, AccessCondition.Object); + + var json = Assert.Single(SavedStrings); + Assert.DoesNotContain("\n", json); + } + + [Fact] + public async Task SerializesVersionsSortedOrder() + { + var newData = new SortedDictionary>(StringComparer.OrdinalIgnoreCase) + { + { + "nuget.versioning", + new SortedSet(StringComparer.OrdinalIgnoreCase) { "nuget", "Microsoft" } + }, + { + "ZDuplicate", + new SortedSet(StringComparer.OrdinalIgnoreCase) { "ownerA", "ownera", "OWNERA", "ownerB" } + }, + { + "EntityFramework", + new SortedSet(StringComparer.OrdinalIgnoreCase) { "Microsoft", "aspnet", "EntityFramework" } + }, + { + "NuGet.Core", + new SortedSet(StringComparer.OrdinalIgnoreCase) { "nuget" } + }, + }; + + await Target.ReplaceLatestIndexedAsync(newData, AccessCondition.Object); + + // Pretty-ify the JSON to make the assertion clearer. + var json = Assert.Single(SavedStrings); + json = JsonConvert.DeserializeObject(json).ToString(); + + Assert.Equal(@"{ + ""EntityFramework"": [ + ""aspnet"", + ""EntityFramework"", + ""Microsoft"" + ], + ""NuGet.Core"": [ + ""nuget"" + ], + ""nuget.versioning"": [ + ""Microsoft"", + ""nuget"" + ], + ""ZDuplicate"": [ + ""ownerA"", + ""ownerB"" + ] +}", json); + + TelemetryService.Verify( + x => x.TrackReplaceLatestIndexedOwners( + /*packageIdCount: */ 4), + Times.Once); + ReplaceLatestIndexedOwnersDurationMetric.Verify(x => x.Dispose(), Times.Once); + } + } + + public class UploadChangeHistoryAsync : Facts + { + public UploadChangeHistoryAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task RejectsEmptyList() + { + var ex = await Assert.ThrowsAsync( + () => Target.UploadChangeHistoryAsync(new string[0])); + Assert.Contains("The list of package IDs must have at least one element.", ex.Message); + } + + [Fact] + public async Task SerializesWithoutBOM() + { + await Target.UploadChangeHistoryAsync(new[] { "nuget" }); + + var bytes = Assert.Single(SavedBytes); + Assert.Equal((byte)'[', bytes[0]); + } + + [Fact] + public async Task SetsContentType() + { + await Target.UploadChangeHistoryAsync(new[] { "nuget" }); + + Assert.Equal("application/json", CloudBlob.Object.Properties.ContentType); + } + + [Fact] + public async Task UsesTimestampAsBlobName() + { + var before = DateTimeOffset.UtcNow; + await Target.UploadChangeHistoryAsync(new[] { "nuget" }); + var after = DateTimeOffset.UtcNow; + + var blobName = Assert.Single(BlobNames); + var slashIndex = blobName.LastIndexOf('/'); + Assert.True(slashIndex >= 0, "The index of the last slash must not be negative."); + + var directoryName = blobName.Substring(0, slashIndex); + Assert.Equal("owners/changes", directoryName); + + var fileName = blobName.Substring(slashIndex + 1); + Assert.EndsWith(".json", fileName); + var timestamp = DateTimeOffset.ParseExact( + fileName, + "yyyy-MM-dd-HH-mm-ss-FFFFFFF\\.\\j\\s\\o\\n", + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal); + Assert.InRange(timestamp, before, after); + } + + [Fact] + public async Task SerializedWithoutIndentation() + { + var input = new[] { "nuget", "Microsoft" }; + + await Target.UploadChangeHistoryAsync(input); + + var json = Assert.Single(SavedStrings); + Assert.DoesNotContain("\n", json); + } + + [Fact] + public async Task SerializesInProvidedOrder() + { + var input = new[] + { + "ZZZ", + "AAA", + "B", + "B", + "z", + "00" + }; + + await Target.UploadChangeHistoryAsync(input); + + // Pretty-ify the JSON to make the assertion clearer. + var json = Assert.Single(SavedStrings); + json = JsonConvert.DeserializeObject(json).ToString(); + Assert.Equal(@"[ + ""ZZZ"", + ""AAA"", + ""B"", + ""B"", + ""z"", + ""00"" +]", json); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + CloudBlobClient = new Mock(); + CloudBlobContainer = new Mock(); + CloudBlob = new Mock(); + Options = new Mock>(); + TelemetryService = new Mock(); + Logger = output.GetLogger(); + Config = new AzureSearchJobConfiguration + { + StorageContainer = "unit-test-container", + }; + + ETag = "\"some-etag\""; + AccessCondition = new Mock(); + ReplaceLatestIndexedOwnersDurationMetric = new Mock(); + + Options + .Setup(x => x.Value) + .Returns(() => Config); + CloudBlobClient + .Setup(x => x.GetContainerReference(It.IsAny())) + .Returns(() => CloudBlobContainer.Object); + CloudBlobContainer + .Setup(x => x.GetBlobReference(It.IsAny())) + .Returns(() => CloudBlob.Object) + .Callback(x => BlobNames.Add(x)); + CloudBlob + .Setup(x => x.ETag) + .Returns(ETag); + CloudBlob + .Setup(x => x.OpenWriteAsync(It.IsAny())) + .ReturnsAsync(() => new RecordingStream(bytes => + { + SavedBytes.Add(bytes); + SavedStrings.Add(Encoding.UTF8.GetString(bytes)); + })); + CloudBlob + .Setup(x => x.Properties) + .Returns(new CloudBlockBlob(new Uri("https://example/blob")).Properties); + + TelemetryService + .Setup(x => x.TrackReplaceLatestIndexedOwners(It.IsAny())) + .Returns(ReplaceLatestIndexedOwnersDurationMetric.Object); + + Target = new OwnerDataClient( + CloudBlobClient.Object, + Options.Object, + TelemetryService.Object, + Logger); + } + + public Mock CloudBlobClient { get; } + public Mock CloudBlobContainer { get; } + public Mock CloudBlob { get; } + public Mock> Options { get; } + public Mock TelemetryService { get; } + public RecordingLogger Logger { get; } + public AzureSearchJobConfiguration Config { get; } + public string ETag { get; } + public Mock AccessCondition { get; } + public Mock ReplaceLatestIndexedOwnersDurationMetric { get; } + public OwnerDataClient Target { get; } + + public List BlobNames { get; } = new List(); + public List SavedBytes { get; } = new List(); + public List SavedStrings { get; } = new List(); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/PopularityTransferDataClientFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/PopularityTransferDataClientFacts.cs new file mode 100644 index 000000000..4b275b116 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/PopularityTransferDataClientFacts.cs @@ -0,0 +1,400 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Moq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class PopularityTransferDataClientFacts + { + public class ReadLatestIndexed : Facts + { + public ReadLatestIndexed(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task AllowsEmptyObject() + { + var json = JsonConvert.SerializeObject(new Dictionary()); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestIndexedAsync(AccessCondition.Object, StringCache); + + Assert.Empty(output.Data); + Assert.Equal(ETag, output.Metadata.ETag); + + TelemetryService.Verify( + x => x.TrackReadLatestIndexedPopularityTransfers( + /*outgoingTransfers: */ 0, + /*modified: */ true, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task AllowsNotModifiedBlob() + { + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ThrowsAsync(new StorageException( + new RequestResult + { + HttpStatusCode = (int)HttpStatusCode.NotModified, + }, + message: "Not modified.", + inner: null)); + + var output = await Target.ReadLatestIndexedAsync(AccessCondition.Object, StringCache); + + Assert.False(output.Modified); + Assert.Null(output.Data); + Assert.Null(output.Metadata); + } + + [Fact] + public async Task ThrowsIfBlobIsMissing() + { + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ThrowsAsync(new StorageException( + new RequestResult + { + HttpStatusCode = (int)HttpStatusCode.NotFound, + }, + message: "Not found.", + inner: null)); + + var ex = await Assert.ThrowsAsync( + () => Target.ReadLatestIndexedAsync(AccessCondition.Object, StringCache)); + + Assert.Equal((int)HttpStatusCode.NotFound, ex.RequestInformation.HttpStatusCode); + } + + [Fact] + public async Task RejectsInvalidJson() + { + var json = JsonConvert.SerializeObject(new object[] + { + new object[] + { + "WindowsAzure.ServiceBus", + new[] { "Azure.Messaging.ServiceBus" } + }, + new object[] + { + "WindowsAzure.Storage", + new[] { "Azure.Storage.Blobs", "Azure.Storage.Queues" } + } + }); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var ex = await Assert.ThrowsAsync( + () => Target.ReadLatestIndexedAsync(AccessCondition.Object, StringCache)); + Assert.Equal("The first token should be the start of an object.", ex.Message); + } + + [Fact] + public async Task ReadsPopularityTransfers() + { + var json = JsonConvert.SerializeObject(new Dictionary + { + { + "windowsazure.servicebus", + new[] { "Azure.Messaging.ServiceBus" } + }, + { + "WindowsAzure.Storage", + new[] { "Azure.Storage.Blobs", "Azure.Storage.Queues" } + }, + { + "ZDuplicate", + new[] { "packageA", "packagea", "PACKAGEA", "packageB" } + }, + }); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestIndexedAsync(AccessCondition.Object, StringCache); + + Assert.Equal(3, output.Data.Count); + Assert.Equal(new[] { "windowsazure.servicebus", "WindowsAzure.Storage", "ZDuplicate" }, output.Data.Keys.ToArray()); + Assert.Equal(new[] { "Azure.Messaging.ServiceBus" }, output.Data["windowsazure.servicebus"].ToArray()); + Assert.Equal(new[] { "Azure.Storage.Blobs", "Azure.Storage.Queues" }, output.Data["WindowsAzure.Storage"].ToArray()); + Assert.Equal(new[] { "packageA", "packageB" }, output.Data["ZDuplicate"].ToArray()); + Assert.Equal(ETag, output.Metadata.ETag); + + CloudBlobContainer.Verify(x => x.GetBlobReference("popularity-transfers/popularity-transfers.v1.json"), Times.Once); + TelemetryService.Verify( + x => x.TrackReadLatestIndexedPopularityTransfers( + /*outgoingTransfers: */ 3, + /*modified: */ true, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task IgnoresEmptyTransferList() + { + var json = JsonConvert.SerializeObject(new Dictionary + { + { + "NoTransfers", + new string[0] + }, + { + "PackageA", + new[] { "PackageB" } + }, + }); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestIndexedAsync(AccessCondition.Object, StringCache); + + Assert.Single(output.Data); + Assert.Equal(new[] { "PackageA" }, output.Data.Keys.ToArray()); + Assert.Equal(new[] { "PackageB" }, output.Data["PackageA"].ToArray()); + Assert.Equal(ETag, output.Metadata.ETag); + + TelemetryService.Verify( + x => x.TrackReadLatestIndexedPopularityTransfers( + /*outgoingTransfers: */ 1, + /*modified: */ true, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task AllowsDuplicateIds() + { + var json = @" +{ + ""PackageA"": [ ""packageB"" ], + ""PackageA"": [ ""packageC"" ], + ""packagea"": [ ""packageD"" ] +}"; + + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestIndexedAsync(AccessCondition.Object, StringCache); + + Assert.Single(output.Data); + Assert.Equal(new[] { "PackageA" }, output.Data.Keys.ToArray()); + Assert.Equal(new[] { "packageB", "packageC", "packageD" }, output.Data["packageA"].ToArray()); + Assert.Equal(ETag, output.Metadata.ETag); + + TelemetryService.Verify( + x => x.TrackReadLatestIndexedPopularityTransfers( + /*outgoingTransfers: */ 1, + /*modified: */ true, + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task DedupesStrings() + { + var json = @" +{ + ""PackageA"": [ ""PackageB"" ], + ""PackageB"": [ ""PackageA"" ] +}"; + + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestIndexedAsync(AccessCondition.Object, StringCache); + + var transfers = output.Data.ToList(); + var transferA = transfers[0]; + var transferB = transfers[1]; + + Assert.Equal(2, StringCache.StringCount); + Assert.Same(transferA.Key, transferB.Value.First()); + Assert.Same(transferB.Key, transferA.Value.First()); + } + } + + public class ReplaceLatestIndexed : Facts + { + public ReplaceLatestIndexed(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task SerializesWithoutBOM() + { + var newData = new PopularityTransferData(); + + await Target.ReplaceLatestIndexedAsync(newData, AccessCondition.Object); + + var bytes = Assert.Single(SavedBytes); + Assert.Equal((byte)'{', bytes[0]); + } + + [Fact] + public async Task SetsContentType() + { + var newData = new PopularityTransferData(); + + await Target.ReplaceLatestIndexedAsync(newData, AccessCondition.Object); + + Assert.Equal("application/json", CloudBlob.Object.Properties.ContentType); + } + + [Fact] + public async Task SerializedWithoutIndentation() + { + var newData = new PopularityTransferData(); + + newData.AddTransfer("PackageA", "packageB"); + newData.AddTransfer("PackageA", "packageC"); + + await Target.ReplaceLatestIndexedAsync(newData, AccessCondition.Object); + + var json = Assert.Single(SavedStrings); + Assert.DoesNotContain("\n", json); + } + + [Fact] + public async Task SerializesVersionsSortedOrder() + { + var newData = new PopularityTransferData(); + + newData.AddTransfer("PackageB", "PackageA"); + newData.AddTransfer("PackageB", "PackageB"); + + newData.AddTransfer("PackageA", "PackageC"); + newData.AddTransfer("PackageA", "packagec"); + newData.AddTransfer("PackageA", "packageC"); + newData.AddTransfer("PackageA", "PackageB"); + + newData.AddTransfer("PackageC", "PackageZ"); + + await Target.ReplaceLatestIndexedAsync(newData, AccessCondition.Object); + + // Pretty-ify the JSON to make the assertion clearer. + var json = Assert.Single(SavedStrings); + json = JsonConvert.DeserializeObject(json).ToString(); + + Assert.Equal(@"{ + ""PackageA"": [ + ""PackageB"", + ""PackageC"" + ], + ""PackageB"": [ + ""PackageA"", + ""PackageB"" + ], + ""PackageC"": [ + ""PackageZ"" + ] +}", json); + TelemetryService.Verify( + x => x.TrackReplaceLatestIndexedPopularityTransfers( + /*outgoingTransfers: */ 3), + Times.Once); + ReplaceLatestIndexedPopularityTransfersDurationMetric.Verify(x => x.Dispose(), Times.Once); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + CloudBlobClient = new Mock(); + CloudBlobContainer = new Mock(); + CloudBlob = new Mock(); + Options = new Mock>(); + TelemetryService = new Mock(); + Logger = output.GetLogger(); + Config = new AzureSearchJobConfiguration + { + StorageContainer = "unit-test-container", + }; + + ETag = "\"some-etag\""; + AccessCondition = new Mock(); + StringCache = new StringCache(); + ReplaceLatestIndexedPopularityTransfersDurationMetric = new Mock(); + + Options + .Setup(x => x.Value) + .Returns(() => Config); + CloudBlobClient + .Setup(x => x.GetContainerReference(It.IsAny())) + .Returns(() => CloudBlobContainer.Object); + CloudBlobContainer + .Setup(x => x.GetBlobReference(It.IsAny())) + .Returns(() => CloudBlob.Object) + .Callback(x => BlobNames.Add(x)); + CloudBlob + .Setup(x => x.ETag) + .Returns(ETag); + CloudBlob + .Setup(x => x.OpenWriteAsync(It.IsAny())) + .ReturnsAsync(() => new RecordingStream(bytes => + { + SavedBytes.Add(bytes); + SavedStrings.Add(Encoding.UTF8.GetString(bytes)); + })); + CloudBlob + .Setup(x => x.Properties) + .Returns(new CloudBlockBlob(new Uri("https://example/blob")).Properties); + + TelemetryService + .Setup(x => x.TrackReplaceLatestIndexedPopularityTransfers(It.IsAny())) + .Returns(ReplaceLatestIndexedPopularityTransfersDurationMetric.Object); + + Target = new PopularityTransferDataClient( + CloudBlobClient.Object, + Options.Object, + TelemetryService.Object, + Logger); + } + + public Mock CloudBlobClient { get; } + public Mock CloudBlobContainer { get; } + public Mock CloudBlob { get; } + public Mock> Options { get; } + public Mock TelemetryService { get; } + public RecordingLogger Logger { get; } + public AzureSearchJobConfiguration Config { get; } + public string ETag { get; } + public Mock AccessCondition { get; } + public StringCache StringCache { get; } + public Mock ReplaceLatestIndexedPopularityTransfersDurationMetric { get; } + public PopularityTransferDataClient Target { get; } + + + public List BlobNames { get; } = new List(); + public List SavedBytes { get; } = new List(); + public List SavedStrings { get; } = new List(); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/StringCacheFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/StringCacheFacts.cs new file mode 100644 index 000000000..7cf196db1 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/StringCacheFacts.cs @@ -0,0 +1,141 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Text; +using Xunit; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class StringCacheFacts + { + public class Dedupe : Facts + { + [Fact] + public void DedupesStrings() + { + var a1 = MakeString("aaa"); + var a2 = MakeString("aaa"); + var a3 = Target.Dedupe(a1); + + var a4 = Target.Dedupe(a2); + + Assert.NotSame(a1, a2); + Assert.Same(a1, a3); + Assert.Same(a1, a4); + } + [Fact] + public void DedupesCaseSensitively() + { + var a1 = MakeString("aaa"); + var a2 = MakeString("AAA"); + + var a3 = Target.Dedupe(a2); + + Assert.NotEqual(a1, a3); + } + } + + public class StringCount : Facts + { + [Fact] + public void CountsUniqueStrings() + { + Target.Dedupe(MakeString("aaa")); + Target.Dedupe(MakeString("bbb")); + Target.Dedupe(MakeString("aaa")); + Target.Dedupe(MakeString("AAA")); + + Assert.Equal(3, Target.StringCount); + } + } + + public class RequestCount : Facts + { + [Fact] + public void CountsNumberOfCalls() + { + Target.Dedupe(MakeString("aaa")); + Target.Dedupe(MakeString("bbb")); + Target.Dedupe(MakeString("aaa")); + Target.Dedupe(MakeString("AAA")); + + Assert.Equal(4, Target.RequestCount); + } + } + + public class HitCount : Facts + { + [Fact] + public void CountsNumberOfDedupedStrings() + { + Target.Dedupe(MakeString("aaa")); + Target.Dedupe(MakeString("bbb")); + Target.Dedupe(MakeString("aaa")); + Target.Dedupe(MakeString("AAA")); + + Assert.Equal(1, Target.HitCount); + } + } + + public class CharCount : Facts + { + [Fact] + public void CountsNumberOfCharactersInDedupedStrings() + { + Target.Dedupe(MakeString("a")); + Target.Dedupe(MakeString("bb")); + Target.Dedupe(MakeString("ccc")); + Target.Dedupe(MakeString("dddd")); + Target.Dedupe(MakeString("a")); + Target.Dedupe(MakeString("bb")); + Target.Dedupe(MakeString("ccc")); + Target.Dedupe(MakeString("dddd")); + + Assert.Equal(10, Target.CharCount); + } + } + + public class ResetCounts : Facts + { + [Fact] + public void CountsNumberOfDedupedStrings() + { + Target.Dedupe(MakeString("aaa")); + Target.Dedupe(MakeString("bbb")); + Target.Dedupe(MakeString("aaa")); + Target.Dedupe(MakeString("AAA")); + + Target.ResetCounts(); + + Assert.Equal(3, Target.StringCount); + Assert.Equal(0, Target.RequestCount); + Assert.Equal(0, Target.HitCount); + Assert.Equal(9, Target.CharCount); + } + } + + public class Facts + { + public Facts() + { + Target = new StringCache(); + } + + public StringCache Target { get; } + + /// + /// Make sure there's no funny compile time string de-duping. + /// + public string MakeString(string input) + { + var sb = new StringBuilder(); + foreach (var c in input) + { + sb.Append(c); + } + + return sb.ToString(); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/VerifiedPackagesDataClientFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/VerifiedPackagesDataClientFacts.cs new file mode 100644 index 000000000..29d36ac3a --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/AuxiliaryFiles/VerifiedPackagesDataClientFacts.cs @@ -0,0 +1,306 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Moq; +using Newtonsoft.Json; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.AuxiliaryFiles +{ + public class VerifiedPackagesDataClientFacts + { + public class ReadLatestAsync : Facts + { + public ReadLatestAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task AllowsEmptyObject() + { + var json = JsonConvert.SerializeObject(new HashSet()); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestAsync(AccessCondition.Object, StringCache); + + Assert.True(output.Modified); + Assert.Empty(output.Data); + Assert.Equal(ETag, output.Metadata.ETag); + } + + [Fact] + public async Task AllowsNotModifiedBlob() + { + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ThrowsAsync(new StorageException( + new RequestResult + { + HttpStatusCode = (int)HttpStatusCode.NotModified, + }, + message: "Not modified.", + inner: null)); + + var output = await Target.ReadLatestAsync(AccessCondition.Object, StringCache); + + Assert.False(output.Modified); + Assert.Null(output.Data); + Assert.Null(output.Metadata); + } + + [Fact] + public async Task RejectsMissingBlob() + { + var expected = new StorageException( + new RequestResult + { + HttpStatusCode = (int)HttpStatusCode.NotFound, + }, + message: "Not found.", + inner: null); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ThrowsAsync(expected); + + var actual = await Assert.ThrowsAsync( + () => Target.ReadLatestAsync(AccessCondition.Object, StringCache)); + Assert.Same(actual, expected); + } + + [Fact] + public async Task RejectsInvalidJson() + { + var json = JsonConvert.SerializeObject(new + { + Version = 7, + Ids = new[] { "nuget.versioning", "EntityFramework" }, + }); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + + var ex = await Assert.ThrowsAsync( + () => Target.ReadLatestAsync(AccessCondition.Object, StringCache)); + Assert.Equal("The first token should be the start of an array.", ex.Message); + } + + [Fact] + public async Task ReadsVerifiedPackages() + { + var json = JsonConvert.SerializeObject(new[] + { + "nuget.versioning", + "EntityFramework", + "NuGet.Core", + }); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestAsync(AccessCondition.Object, StringCache); + + Assert.True(output.Modified); + Assert.Equal(new[] { "EntityFramework", "NuGet.Core", "nuget.versioning" }, output.Data.OrderBy(x => x).ToArray()); + Assert.Equal(ETag, output.Metadata.ETag); + CloudBlobContainer.Verify(x => x.GetBlobReference("verified-packages/verified-packages.v1.json"), Times.Once); + } + + [Fact] + public async Task AllowsDuplicateIdsWithDifferentCase() + { + var json = JsonConvert.SerializeObject(new[] + { + "NuGet.Core", + "nuget.core", + }); + CloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(Encoding.UTF8.GetBytes(json))); + + var output = await Target.ReadLatestAsync(AccessCondition.Object, StringCache); + + Assert.True(output.Modified); + Assert.Single(output.Data); + Assert.Equal(new[] { "NuGet.Core" }, output.Data.ToArray()); + Assert.Equal(ETag, output.Metadata.ETag); + } + } + + public class ReplaceLatestAsync : Facts + { + public ReplaceLatestAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task SerializesWithoutBOM() + { + var newData = new HashSet(); + + await Target.ReplaceLatestAsync(newData, AccessCondition.Object); + + var bytes = Assert.Single(SavedBytes); + Assert.Equal((byte)'[', bytes[0]); + } + + [Fact] + public async Task SetsContentType() + { + var newData = new HashSet(); + + await Target.ReplaceLatestAsync(newData, AccessCondition.Object); + + Assert.Equal("application/json", CloudBlob.Object.Properties.ContentType); + } + + [Fact] + public async Task SerializedWithoutIndentation() + { + var newData = new HashSet + { + "nuget.versioning", + "NuGet.Core", + }; + + await Target.ReplaceLatestAsync(newData, AccessCondition.Object); + + var json = Assert.Single(SavedStrings); + Assert.DoesNotContain("\n", json); + } + + [Fact] + public async Task SerializesPackageIds() + { + var newData = new HashSet + { + "nuget.versioning", + "EntityFramework", + "NuGet.Core", + }; + + await Target.ReplaceLatestAsync(newData, AccessCondition.Object); + + // Pretty-ify and sort the JSON to make the assertion clearer. + var json = Assert.Single(SavedStrings); + var array = JsonConvert + .DeserializeObject(json) + .OrderBy(x => x, StringComparer.OrdinalIgnoreCase) + .ToArray(); + json = JsonConvert.SerializeObject(array, Formatting.Indented); + + Assert.Equal(@"[ + ""EntityFramework"", + ""NuGet.Core"", + ""nuget.versioning"" +]", json); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + CloudBlobClient = new Mock(); + CloudBlobContainer = new Mock(); + CloudBlob = new Mock(); + Options = new Mock>(); + TelemetryService = new Mock(); + Logger = output.GetLogger(); + Config = new AzureSearchConfiguration + { + StorageContainer = "unit-test-container", + }; + + ETag = "\"some-etag\""; + AccessCondition = new Mock(); + StringCache = new StringCache(); + + Options + .Setup(x => x.Value) + .Returns(() => Config); + CloudBlobClient + .Setup(x => x.GetContainerReference(It.IsAny())) + .Returns(() => CloudBlobContainer.Object); + CloudBlobContainer + .Setup(x => x.GetBlobReference(It.IsAny())) + .Returns(() => CloudBlob.Object) + .Callback(x => BlobNames.Add(x)); + CloudBlob + .Setup(x => x.ETag) + .Returns(ETag); + CloudBlob + .Setup(x => x.OpenWriteAsync(It.IsAny())) + .ReturnsAsync(() => new RecordingStream(bytes => + { + SavedBytes.Add(bytes); + SavedStrings.Add(Encoding.UTF8.GetString(bytes)); + })); + CloudBlob + .Setup(x => x.Properties) + .Returns(new CloudBlockBlob(new Uri("https://example/blob")).Properties); + + Target = new VerifiedPackagesDataClient( + CloudBlobClient.Object, + Options.Object, + TelemetryService.Object, + Logger); + } + + public Mock CloudBlobClient { get; } + public Mock CloudBlobContainer { get; } + public Mock CloudBlob { get; } + public Mock> Options { get; } + public Mock TelemetryService { get; } + public RecordingLogger Logger { get; } + public AzureSearchConfiguration Config { get; } + public string ETag { get; } + public Mock AccessCondition { get; } + public StringCache StringCache { get; } + public VerifiedPackagesDataClient Target { get; } + + public List BlobNames { get; } = new List(); + public List SavedBytes { get; } = new List(); + public List SavedStrings { get; } = new List(); + } + + private class RecordingStream : MemoryStream + { + private readonly object _lock = new object(); + private Action _onDispose; + + public RecordingStream(Action onDispose) + { + _onDispose = onDispose; + } + + protected override void Dispose(bool disposing) + { + lock (_lock) + { + if (_onDispose != null) + { + _onDispose(ToArray()); + _onDispose = null; + } + } + + base.Dispose(disposing); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/BaseDocumentBuilderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/BaseDocumentBuilderFacts.cs new file mode 100644 index 000000000..770c21eca --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/BaseDocumentBuilderFacts.cs @@ -0,0 +1,329 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Protocol.Catalog; +using NuGet.Services.AzureSearch.Support; +using NuGet.Services.Entities; +using Xunit; + +namespace NuGet.Services.AzureSearch +{ + public class BaseDocumentBuilderFacts + { + public class PopulateMetadataWithCatalogLeaf : Facts + { + [Theory] + [InlineData("any")] + [InlineData("agnostic")] + [InlineData("unsupported")] + [InlineData("fakeframework1.0")] + public void DoesNotIncludeDependencyVersionSpecialFrameworks(string framework) + { + var leaf = new PackageDetailsCatalogLeaf + { + PackageId = Data.PackageId, + PackageVersion = Data.NormalizedVersion, + DependencyGroups = new List + { + new PackageDependencyGroup + { + TargetFramework = framework, + Dependencies = new List + { + new Protocol.Catalog.PackageDependency + { + Id = "NuGet.Versioning", + Range = "2.0.0", + }, + new Protocol.Catalog.PackageDependency + { + Id = "NuGet.Frameworks", + Range = "3.0.0", + }, + }, + }, + }, + }; + var full = new HijackDocument.Full(); + + Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); + + Assert.Equal("NuGet.Versioning:2.0.0|NuGet.Frameworks:3.0.0", full.FlattenedDependencies); + } + + [Theory] + [InlineData("any", ":")] + [InlineData("net40", "::net40")] + [InlineData("NET45", "::net45")] + [InlineData(".NETFramework,Version=4.0", "::net40")] + public void AddsEmptyDependencyGroup(string framework, string expected) + { + var leaf = new PackageDetailsCatalogLeaf + { + PackageId = Data.PackageId, + PackageVersion = Data.NormalizedVersion, + DependencyGroups = new List + { + new PackageDependencyGroup + { + TargetFramework = framework, + }, + }, + }; + var full = new HijackDocument.Full(); + + Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); + + Assert.Equal(expected, full.FlattenedDependencies); + } + + [Fact] + public void AddsEmptyStringForDependencyVersionAllRange() + { + var leaf = new PackageDetailsCatalogLeaf + { + PackageId = Data.PackageId, + PackageVersion = Data.NormalizedVersion, + DependencyGroups = new List + { + new PackageDependencyGroup + { + TargetFramework = "net40", + Dependencies = new List + { + new Protocol.Catalog.PackageDependency + { + Id = "NuGet.Versioning", + Range = "(, )" + } + }, + }, + }, + }; + var full = new HijackDocument.Full(); + + Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); + + Assert.Equal("NuGet.Versioning::net40", full.FlattenedDependencies); + } + + [Theory] + [InlineData("1.0.0", "1.0.0")] + [InlineData("1.0.0-beta+git", "1.0.0-beta")] + [InlineData("[1.0.0-beta+git, )", "1.0.0-beta")] + [InlineData("[1.0.0, 1.0.0]", "[1.0.0]")] + [InlineData("[1.0.0, 2.0.0)", "[1.0.0, 2.0.0)")] + [InlineData("[1.0.0, )", "1.0.0")] + public void AddShortFormOfDependencyVersionRange(string input, string expected) + { + var leaf = new PackageDetailsCatalogLeaf + { + PackageId = Data.PackageId, + PackageVersion = Data.NormalizedVersion, + DependencyGroups = new List + { + new PackageDependencyGroup + { + TargetFramework = "net40", + Dependencies = new List + { + new Protocol.Catalog.PackageDependency + { + Id = "NuGet.Versioning", + Range = input + } + }, + }, + }, + }; + var full = new HijackDocument.Full(); + + Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); + + Assert.Equal("NuGet.Versioning:" + expected + ":net40", full.FlattenedDependencies); + } + + [Fact] + public void AllowNullDependencyGroups() + { + var leaf = new PackageDetailsCatalogLeaf + { + PackageId = Data.PackageId, + PackageVersion = Data.NormalizedVersion, + DependencyGroups = null + }; + var full = new HijackDocument.Full(); + + Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); + + Assert.Null(full.FlattenedDependencies); + } + + [Fact] + public void IfLeafHasIconFile_LinksToFlatContainer() + { + var leaf = new PackageDetailsCatalogLeaf + { + PackageId = Data.PackageId, + PackageVersion = Data.NormalizedVersion, + IconFile = "iconFile", + IconUrl = "iconUrl" + }; + + var full = new HijackDocument.Full(); + + Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); + + Assert.Equal(Data.FlatContainerIconUrl, full.IconUrl); + } + + [Fact] + public void IfLeafDoesNotHaveIconFile_UsesIconUrl() + { + var iconUrl = "iconUrl"; + var leaf = new PackageDetailsCatalogLeaf + { + PackageId = Data.PackageId, + PackageVersion = Data.NormalizedVersion, + IconUrl = iconUrl + }; + + var full = new HijackDocument.Full(); + + Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); + + Assert.Equal(iconUrl, full.IconUrl); + } + + [Fact] + public void IfLeafDoesNotHaveIconFileButHasUrl_UsesFlatContainerIfConfigured() + { + Config.AllIconsInFlatContainer = true; + var leaf = new PackageDetailsCatalogLeaf + { + PackageId = Data.PackageId, + PackageVersion = Data.NormalizedVersion, + IconUrl = "iconUrl" + }; + + var full = new HijackDocument.Full(); + + Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); + + Assert.Equal(Data.FlatContainerIconUrl, full.IconUrl); + } + + [Fact] + public void IfLeafDoesNotHaveAnyIconFile_NoIconUrlIsSet() + { + Config.AllIconsInFlatContainer = true; + var leaf = new PackageDetailsCatalogLeaf + { + PackageId = Data.PackageId, + PackageVersion = Data.NormalizedVersion + }; + + var full = new HijackDocument.Full(); + + Target.PopulateMetadata(full, Data.NormalizedVersion, leaf); + + Assert.Null(full.IconUrl); + } + } + + public class PopulateMetadataWithPackage : Facts + { + [Fact] + public void IfPackageHasEmbeddedIcon_LinksToFlatContainer() + { + var package = new Package + { + NormalizedVersion = Data.NormalizedVersion, + IconUrl = "iconUrl", + HasEmbeddedIcon = true + }; + + var full = new HijackDocument.Full(); + + Target.PopulateMetadata(full, Data.PackageId, package); + + Assert.Equal(Data.FlatContainerIconUrl, full.IconUrl); + } + + [Fact] + public void IfPackageDoesNotHaveEmbeddedIcon_UsesIconUrl() + { + var iconUrl = "iconUrl"; + var package = new Package + { + NormalizedVersion = Data.NormalizedVersion, + IconUrl = iconUrl + }; + + var full = new HijackDocument.Full(); + + Target.PopulateMetadata(full, Data.PackageId, package); + + Assert.Equal(iconUrl, full.IconUrl); + } + + [Fact] + public void IfPackageDoesNotHaveIconFileButHasUrl_UsesFlatContainerIfConfigured() + { + Config.AllIconsInFlatContainer = true; + var package = new Package + { + NormalizedVersion = Data.NormalizedVersion, + IconUrl = "iconUrl" + }; + + var full = new HijackDocument.Full(); + + Target.PopulateMetadata(full, Data.PackageId, package); + + Assert.Equal(Data.FlatContainerIconUrl, full.IconUrl); + } + + [Fact] + public void IfPackageDoesNotHaveAnyIconFile_NoIconUrlIsSet() + { + Config.AllIconsInFlatContainer = true; + var package = new Package + { + NormalizedVersion = Data.NormalizedVersion, + }; + + var full = new HijackDocument.Full(); + + Target.PopulateMetadata(full, Data.PackageId, package); + + Assert.Null(full.IconUrl); + } + } + + public abstract class Facts + { + public Facts() + { + Options = new Mock>(); + Config = new AzureSearchJobConfiguration + { + GalleryBaseUrl = Data.GalleryBaseUrl, + FlatContainerBaseUrl = Data.FlatContainerBaseUrl, + FlatContainerContainerName = Data.FlatContainerContainerName, + }; + + Options.Setup(o => o.Value).Returns(() => Config); + + Target = new BaseDocumentBuilder(Options.Object); + } + + public Mock> Options { get; } + public AzureSearchJobConfiguration Config { get; } + public BaseDocumentBuilder Target { get; } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/BatchPusherFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/BatchPusherFacts.cs new file mode 100644 index 000000000..dc0e1d92d --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/BatchPusherFacts.cs @@ -0,0 +1,753 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Options; +using Microsoft.Rest; +using Microsoft.Rest.Azure; +using Moq; +using NuGet.Services.AzureSearch.Wrappers; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch +{ + public class BatchPusherFacts + { + public class FinishAsync : BaseFacts + { + public FinishAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task ReturnsFailedPackageIds() + { + _versionListDataClient + .Setup(x => x.TryReplaceAsync(IdB, It.IsAny(), It.IsAny())) + .ReturnsAsync(false); + + _config.AzureSearchBatchSize = 7; + _target.EnqueueIndexActions(IdA, _indexActions); + _target.EnqueueIndexActions(IdB, _indexActions); + _target.EnqueueIndexActions(IdC, _indexActions); + + var result = await _target.TryFinishAsync(); + + Assert.Equal(new[] { IdB }, result.FailedPackageIds.ToArray()); + + Assert.Equal(3, _hijackBatches.Count); + Assert.Equal(2, _searchBatches.Count); + + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdA, + It.IsAny(), + It.IsAny()), + Times.Once); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdB, + It.IsAny(), + It.IsAny()), + Times.Once); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdC, + It.IsAny(), + It.IsAny()), + Times.Once); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Exactly(3)); + } + + [Fact] + public async Task UpdatesOnlyAllVersionLists() + { + _config.AzureSearchBatchSize = 7; + _target.EnqueueIndexActions(IdA, _indexActions); + _target.EnqueueIndexActions(IdB, _indexActions); + _target.EnqueueIndexActions(IdC, _indexActions); + + var result = await _target.TryFinishAsync(); + + Assert.Empty(result.FailedPackageIds); + + Assert.Equal(3, _hijackBatches.Count); + Assert.Equal(2, _searchBatches.Count); + + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdA, + It.IsAny(), + It.IsAny()), + Times.Once); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdB, + It.IsAny(), + It.IsAny()), + Times.Once); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdC, + It.IsAny(), + It.IsAny()), + Times.Once); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Exactly(3)); + } + + [Fact] + public async Task UpdatesVersionListIfAllActionsAreDone() + { + _config.AzureSearchBatchSize = 1; + _target.EnqueueIndexActions(IdA, _indexActions); + + var result = await _target.TryFinishAsync(); + + Assert.Empty(result.FailedPackageIds); + + Assert.Equal(5, _hijackBatches.Count); + Assert.Equal(3, _searchBatches.Count); + + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdA, + _indexActions.VersionListDataResult.Result, + _indexActions.VersionListDataResult.AccessCondition), + Times.Once); + } + + [Fact] + public async Task SkipsUpdatingVersionListIfDisabled() + { + _config.AzureSearchBatchSize = 1; + _developmentConfig.DisableVersionListWriters = true; + _target.EnqueueIndexActions(IdA, _indexActions); + + var result = await _target.TryFinishAsync(); + + Assert.Empty(result.FailedPackageIds); + + Assert.Equal(5, _hijackBatches.Count); + Assert.Equal(3, _searchBatches.Count); + + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task PushesPartialBatch() + { + _config.AzureSearchBatchSize = 1000; + _target.EnqueueIndexActions(IdA, _indexActions); + + var result = await _target.TryFinishAsync(); + + Assert.Empty(result.FailedPackageIds); + + Assert.Single(_hijackBatches); + Assert.Equal(_hijackDocuments, _hijackBatches[0].Actions.ToArray()); + Assert.Single(_searchBatches); + Assert.Equal(_searchDocuments, _searchBatches[0].Actions.ToArray()); + + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdA, + _indexActions.VersionListDataResult.Result, + _indexActions.VersionListDataResult.AccessCondition), + Times.Once); + + Assert.Empty(_target._versionListDataResults); + Assert.Empty(_target._hijackActions); + Assert.Empty(_target._searchActions); + Assert.Empty(_target._idReferenceCount); + } + + [Fact] + public async Task PushesFullBatchesThenPartialBatch() + { + _config.AzureSearchBatchSize = 2; + _target.EnqueueIndexActions(IdA, _indexActions); + + var result = await _target.TryFinishAsync(); + + Assert.Empty(result.FailedPackageIds); + + Assert.Equal(3, _hijackBatches.Count); + Assert.Equal(new[] { _hijackDocumentA, _hijackDocumentB }, _hijackBatches[0].Actions.ToArray()); + Assert.Equal(new[] { _hijackDocumentC, _hijackDocumentD }, _hijackBatches[1].Actions.ToArray()); + Assert.Equal(new[] { _hijackDocumentE }, _hijackBatches[2].Actions.ToArray()); + Assert.Equal(2, _searchBatches.Count); + Assert.Equal(new[] { _searchDocumentA, _searchDocumentB }, _searchBatches[0].Actions.ToArray()); + Assert.Equal(new[] { _searchDocumentC }, _searchBatches[1].Actions.ToArray()); + + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdA, + _indexActions.VersionListDataResult.Result, + _indexActions.VersionListDataResult.AccessCondition), + Times.Once); + + Assert.Empty(_target._versionListDataResults); + Assert.Empty(_target._hijackActions); + Assert.Empty(_target._searchActions); + Assert.Empty(_target._idReferenceCount); + } + + [Fact] + public async Task SplitsBatchesInHalfWhenTooLarge() + { + _config.AzureSearchBatchSize = 100; + _target.EnqueueIndexActions(IdA, _indexActions); + _hijackDocumentsWrapper + .Setup(x => x.IndexAsync(It.IsAny>())) + .Returns>(b => + { + _hijackBatches.Add(b); + if (b.Actions.Count() > 1) + { + throw new CloudException + { + Response = new HttpResponseMessageWrapper( + new HttpResponseMessage(HttpStatusCode.RequestEntityTooLarge), + "Too big!"), + }; + } + + return Task.FromResult(new DocumentIndexResult(new List())); + }); + + var result = await _target.TryFinishAsync(); + + Assert.Empty(result.FailedPackageIds); + + Assert.Equal(9, _hijackBatches.Count); + Assert.Equal( + new[] { _hijackDocumentA, _hijackDocumentB, _hijackDocumentC, _hijackDocumentD, _hijackDocumentE }, + _hijackBatches[0].Actions.ToArray()); + Assert.Equal( + new[] { _hijackDocumentA, _hijackDocumentB }, + _hijackBatches[1].Actions.ToArray()); + Assert.Equal( + new[] { _hijackDocumentA }, + _hijackBatches[2].Actions.ToArray()); + Assert.Equal( + new[] { _hijackDocumentB }, + _hijackBatches[3].Actions.ToArray()); + Assert.Equal( + new[] { _hijackDocumentC, _hijackDocumentD, _hijackDocumentE }, + _hijackBatches[4].Actions.ToArray()); + Assert.Equal( + new[] { _hijackDocumentC }, + _hijackBatches[5].Actions.ToArray()); + Assert.Equal( + new[] { _hijackDocumentD, _hijackDocumentE }, + _hijackBatches[6].Actions.ToArray()); + Assert.Equal( + new[] { _hijackDocumentD }, + _hijackBatches[7].Actions.ToArray()); + Assert.Equal( + new[] { _hijackDocumentE }, + _hijackBatches[8].Actions.ToArray()); + } + + [Fact] + public async Task StopsSplittingBatchesAtOne() + { + _config.AzureSearchBatchSize = 100; + _target.EnqueueIndexActions(IdA, _indexActions); + _hijackDocumentsWrapper + .Setup(x => x.IndexAsync(It.IsAny>())) + .Returns>(b => + { + _hijackBatches.Add(b); + throw new CloudException + { + Response = new HttpResponseMessageWrapper( + new HttpResponseMessage(HttpStatusCode.RequestEntityTooLarge), + "Too big!"), + }; + }); + + var ex = await Assert.ThrowsAsync( + () => _target.TryFinishAsync()); + + Assert.Equal(HttpStatusCode.RequestEntityTooLarge, ex.Response.StatusCode); + Assert.Equal(3, _hijackBatches.Count); + Assert.Equal( + new[] { _hijackDocumentA, _hijackDocumentB, _hijackDocumentC, _hijackDocumentD, _hijackDocumentE }, + _hijackBatches[0].Actions.ToArray()); + Assert.Equal( + new[] { _hijackDocumentA, _hijackDocumentB }, + _hijackBatches[1].Actions.ToArray()); + Assert.Equal( + new[] { _hijackDocumentA }, + _hijackBatches[2].Actions.ToArray()); + } + } + + public class PushFullBatchesAsync : BaseFacts + { + public PushFullBatchesAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task LogsUpALimitedNumberOfFailedResults() + { + _target.EnqueueIndexActions(IdA, _indexActions); + _searchDocumentsWrapper + .Setup(x => x.IndexAsync(It.IsAny>())) + .ReturnsAsync(() => new DocumentIndexResult(new List + { + new IndexingResult(key: "A-0", errorMessage: "A-0 message", succeeded: false, statusCode: 0), + new IndexingResult(key: "A-1", errorMessage: "A-1 message", succeeded: false, statusCode: 1), + new IndexingResult(key: "A-2", errorMessage: "A-2 message", succeeded: true, statusCode: 2), + new IndexingResult(key: "A-3", errorMessage: "A-3 message", succeeded: false, statusCode: 3), + new IndexingResult(key: "A-4", errorMessage: "A-4 message", succeeded: false, statusCode: 4), + new IndexingResult(key: "A-5", errorMessage: "A-5 message", succeeded: false, statusCode: 5), + new IndexingResult(key: "A-6", errorMessage: "A-6 message", succeeded: false, statusCode: 6), + })); + + var ex = await Assert.ThrowsAsync( + () => _target.TryPushFullBatchesAsync()); + + Assert.Contains("Errors were found when indexing a batch. Up to 5 errors get logged.", ex.Message); + Assert.Contains("Indexing document with key A-0 failed for index search. 0: A-0 message", _logger.Messages); + Assert.Contains("Indexing document with key A-1 failed for index search. 1: A-1 message", _logger.Messages); + Assert.Contains("Indexing document with key A-3 failed for index search. 3: A-3 message", _logger.Messages); + Assert.Contains("Indexing document with key A-4 failed for index search. 4: A-4 message", _logger.Messages); + Assert.Contains("Indexing document with key A-5 failed for index search. 5: A-5 message", _logger.Messages); + + Assert.All(_logger.Messages, x => Assert.DoesNotContain("A-2", x)); + Assert.All(_logger.Messages, x => Assert.DoesNotContain("A-6", x)); + + Assert.Null(ex.InnerException); + } + + [Fact] + public async Task ReturnsFailedPackageIds() + { + _versionListDataClient + .Setup(x => x.TryReplaceAsync(IdA, It.IsAny(), It.IsAny())) + .ReturnsAsync(false); + + _config.AzureSearchBatchSize = 7; + _target.EnqueueIndexActions(IdA, _indexActions); + _target.EnqueueIndexActions(IdB, _indexActions); + _target.EnqueueIndexActions(IdC, _indexActions); + + var result = await _target.TryPushFullBatchesAsync(); + + Assert.Equal(new[] { IdA }, result.FailedPackageIds.ToArray()); + + Assert.Equal(2, _hijackBatches.Count); + Assert.Single(_searchBatches); + + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdA, + It.IsAny(), + It.IsAny()), + Times.Once); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdB, + It.IsAny(), + It.IsAny()), + Times.Once); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdC, + It.IsAny(), + It.IsAny()), + Times.Never); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Exactly(2)); + } + + [Fact] + public async Task UpdatesOnlyFinishedVersionLists() + { + _config.AzureSearchBatchSize = 7; + _target.EnqueueIndexActions(IdA, _indexActions); + _target.EnqueueIndexActions(IdB, _indexActions); + _target.EnqueueIndexActions(IdC, _indexActions); + + var result = await _target.TryPushFullBatchesAsync(); + + Assert.Empty(result.FailedPackageIds); + + Assert.Equal(2, _hijackBatches.Count); + Assert.Single(_searchBatches); + + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdA, + It.IsAny(), + It.IsAny()), + Times.Once); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdB, + It.IsAny(), + It.IsAny()), + Times.Once); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdC, + It.IsAny(), + It.IsAny()), + Times.Never); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Exactly(2)); + } + + [Fact] + public async Task OnlyPushesFullBatches() + { + _config.AzureSearchBatchSize = 2; + _target.EnqueueIndexActions(IdA, _indexActions); + + var result = await _target.TryPushFullBatchesAsync(); + + Assert.Empty(result.FailedPackageIds); + + Assert.Equal(2, _hijackBatches.Count); + Assert.Equal(new[] { _hijackDocumentA, _hijackDocumentB }, _hijackBatches[0].Actions.ToArray()); + Assert.Equal(new[] { _hijackDocumentC, _hijackDocumentD }, _hijackBatches[1].Actions.ToArray()); + Assert.Single(_searchBatches); + Assert.Equal(new[] { _searchDocumentA, _searchDocumentB }, _searchBatches[0].Actions.ToArray()); + + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + + Assert.Single(_target._versionListDataResults); + Assert.Equal(new[] { _hijackDocumentE }, _target._hijackActions.Select(x => x.Value).ToArray()); + Assert.Equal(new[] { _searchDocumentC }, _target._searchActions.Select(x => x.Value).ToArray()); + Assert.Equal(2, _target._idReferenceCount[IdA]); + } + + [Fact] + public async Task AllowsPushingNoBatchesIfNoBatchesAreFull() + { + _config.AzureSearchBatchSize = 1000; + _target.EnqueueIndexActions(IdA, _indexActions); + + var result = await _target.TryPushFullBatchesAsync(); + + Assert.Empty(result.FailedPackageIds); + + Assert.Empty(_hijackBatches); + Assert.Empty(_searchBatches); + + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task DoesNotUpdateVersionListIfOnlyOneIndexsActionsAreComplete() + { + _config.AzureSearchBatchSize = 3; + _target.EnqueueIndexActions(IdA, _indexActions); + + var result = await _target.TryPushFullBatchesAsync(); + + Assert.Empty(result.FailedPackageIds); + + Assert.Single(_hijackBatches); + Assert.Equal(new[] { _hijackDocumentA, _hijackDocumentB, _hijackDocumentC }, _hijackBatches[0].Actions.ToArray()); + Assert.Single(_searchBatches); + Assert.Equal(new[] { _searchDocumentA, _searchDocumentB, _searchDocumentC }, _searchBatches[0].Actions.ToArray()); + + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task UpdatesVersionListIfAllActionsAreDone() + { + _config.AzureSearchBatchSize = 1; + _target.EnqueueIndexActions(IdA, _indexActions); + + var result = await _target.TryPushFullBatchesAsync(); + + Assert.Empty(result.FailedPackageIds); + + Assert.Equal(5, _hijackBatches.Count); + Assert.Equal(3, _searchBatches.Count); + + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdA, + _indexActions.VersionListDataResult.Result, + _indexActions.VersionListDataResult.AccessCondition), + Times.Once); + } + + [Fact] + public async Task AllowsReprocessingCompletedActions() + { + _config.AzureSearchBatchSize = 1; + _target.EnqueueIndexActions(IdA, _indexActions); + await _target.TryPushFullBatchesAsync(); + _target.EnqueueIndexActions(IdA, _indexActions); + + var result = await _target.TryPushFullBatchesAsync(); + + Assert.Empty(result.FailedPackageIds); + + Assert.Equal(10, _hijackBatches.Count); + Assert.Equal(6, _searchBatches.Count); + + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Exactly(2)); + _versionListDataClient.Verify( + x => x.TryReplaceAsync( + IdA, + _indexActions.VersionListDataResult.Result, + _indexActions.VersionListDataResult.AccessCondition), + Times.Exactly(2)); + } + } + + public class EnqueueIndexActions : BaseFacts + { + public EnqueueIndexActions(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public void RejectsDuplicateId() + { + _target.EnqueueIndexActions(IdA, _indexActions); + + var ex = Assert.Throws( + () => _target.EnqueueIndexActions(IdA, _indexActions)); + Assert.Contains("This package ID has already been enqueued.", ex.Message); + } + + [Fact] + public void EnqueuesAndIncrements() + { + _target.EnqueueIndexActions(IdA, _indexActions); + + Assert.Equal(3, _target._searchActions.Count); + Assert.Equal(_searchDocuments, _target._searchActions.Select(x => x.Value).ToList()); + Assert.All(_target._searchActions, x => Assert.Equal(IdA, x.Id)); + + Assert.Equal(5, _target._hijackActions.Count); + Assert.Equal(_hijackDocuments, _target._hijackActions.Select(x => x.Value).ToList()); + Assert.All(_target._hijackActions, x => Assert.Equal(IdA, x.Id)); + + Assert.Equal(new[] { IdA }, _target._idReferenceCount.Keys.ToArray()); + Assert.Equal(8, _target._idReferenceCount[IdA]); + + Assert.Equal(new[] { IdA }, _target._versionListDataResults.Keys.ToArray()); + Assert.Same(_indexActions.VersionListDataResult, _target._versionListDataResults[IdA]); + } + + [Fact] + public void RejectsEmptyEnqueue() + { + var emptyIndexActions = new IndexActions( + new List>(), + new List>(), + new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + AccessConditionWrapper.GenerateEmptyCondition())); + + var ex = Assert.Throws( + () => _target.EnqueueIndexActions(IdA, emptyIndexActions)); + Assert.Contains("There must be at least one index action.", ex.Message); + Assert.Empty(_target._searchActions); + Assert.Empty(_target._hijackActions); + Assert.Empty(_target._idReferenceCount); + Assert.Empty(_target._versionListDataResults); + } + } + + public abstract class BaseFacts + { + protected const string IdA = "NuGet.Versioning"; + protected const string IdB = "NuGet.Frameworks"; + protected const string IdC = "NuGet.Packaging"; + + protected readonly RecordingLogger _logger; + protected readonly Mock _searchIndexClientWrapper; + protected readonly Mock _searchDocumentsWrapper; + protected readonly Mock _hijackIndexClientWrapper; + protected readonly Mock _hijackDocumentsWrapper; + protected readonly Mock _versionListDataClient; + protected readonly AzureSearchJobConfiguration _config; + protected readonly AzureSearchJobDevelopmentConfiguration _developmentConfig; + protected readonly Mock> _options; + protected readonly Mock> _developmentOptions; + protected readonly Mock _telemetryService; + protected readonly IndexActions _indexActions; + protected readonly BatchPusher _target; + + protected readonly IndexAction _searchDocumentA; + protected readonly IndexAction _searchDocumentB; + protected readonly IndexAction _searchDocumentC; + protected readonly List> _searchDocuments; + protected readonly IndexAction _hijackDocumentA; + protected readonly IndexAction _hijackDocumentB; + protected readonly IndexAction _hijackDocumentC; + protected readonly IndexAction _hijackDocumentD; + protected readonly IndexAction _hijackDocumentE; + protected readonly List> _hijackDocuments; + protected readonly List> _searchBatches; + protected readonly List> _hijackBatches; + + public BaseFacts(ITestOutputHelper output) + { + _logger = output.GetLogger(); + _searchIndexClientWrapper = new Mock(); + _searchDocumentsWrapper = new Mock(); + _hijackIndexClientWrapper = new Mock(); + _hijackDocumentsWrapper = new Mock(); + _versionListDataClient = new Mock(); + _config = new AzureSearchJobConfiguration(); + _developmentConfig = new AzureSearchJobDevelopmentConfiguration(); + _options = new Mock>(); + _developmentOptions = new Mock>(); + _telemetryService = new Mock(); + + _searchIndexClientWrapper.Setup(x => x.IndexName).Returns("search"); + _searchIndexClientWrapper.Setup(x => x.Documents).Returns(() => _searchDocumentsWrapper.Object); + _hijackIndexClientWrapper.Setup(x => x.IndexName).Returns("hijack"); + _hijackIndexClientWrapper.Setup(x => x.Documents).Returns(() => _hijackDocumentsWrapper.Object); + _versionListDataClient + .Setup(x => x.TryReplaceAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(true); + _options.Setup(x => x.Value).Returns(() => _config); + _developmentOptions.Setup(x => x.Value).Returns(() => _developmentConfig); + + _searchBatches = new List>(); + _hijackBatches = new List>(); + + _searchDocumentsWrapper + .Setup(x => x.IndexAsync(It.IsAny>())) + .ReturnsAsync(() => new DocumentIndexResult(new List())) + .Callback>(b => _searchBatches.Add(b)); + _hijackDocumentsWrapper + .Setup(x => x.IndexAsync(It.IsAny>())) + .ReturnsAsync(() => new DocumentIndexResult(new List())) + .Callback>(b => _hijackBatches.Add(b)); + + _config.AzureSearchBatchSize = 2; + _config.MaxConcurrentVersionListWriters = 1; + + _searchDocumentA = IndexAction.Upload(new KeyedDocument()); + _searchDocumentB = IndexAction.Upload(new KeyedDocument()); + _searchDocumentC = IndexAction.Upload(new KeyedDocument()); + _searchDocuments = new List> + { + _searchDocumentA, + _searchDocumentB, + _searchDocumentC, + }; + + _hijackDocumentA = IndexAction.Upload(new KeyedDocument()); + _hijackDocumentB = IndexAction.Upload(new KeyedDocument()); + _hijackDocumentC = IndexAction.Upload(new KeyedDocument()); + _hijackDocumentD = IndexAction.Upload(new KeyedDocument()); + _hijackDocumentE = IndexAction.Upload(new KeyedDocument()); + _hijackDocuments = new List> + { + _hijackDocumentA, + _hijackDocumentB, + _hijackDocumentC, + _hijackDocumentD, + _hijackDocumentE, + }; + + _indexActions = new IndexActions( + _searchDocuments, + _hijackDocuments, + new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + AccessConditionWrapper.GenerateEmptyCondition())); + + _target = new BatchPusher( + _searchIndexClientWrapper.Object, + _hijackIndexClientWrapper.Object, + _versionListDataClient.Object, + _options.Object, + _developmentOptions.Object, + _telemetryService.Object, + _logger); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/AzureSearchCollectorLogicFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/AzureSearchCollectorLogicFacts.cs new file mode 100644 index 000000000..f01904835 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/AzureSearchCollectorLogicFacts.cs @@ -0,0 +1,567 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Packaging.Core; +using NuGet.Protocol.Catalog; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.V3; +using NuGet.Versioning; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public class AzureSearchCollectorLogicFacts + { + public class CreateBatchesAsync : BaseFacts + { + public CreateBatchesAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task SingleBatchWithAllItems() + { + var items = new[] + { + new CatalogCommitItem( + uri: null, + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List(), + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + new CatalogCommitItem( + uri: null, + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 2), + types: null, + typeUris: new List(), + packageIdentity: new PackageIdentity("NuGet.Frameworks", NuGetVersion.Parse("2.0.0"))), + }; + + var batches = await _target.CreateBatchesAsync(items); + + var batch = Assert.Single(batches); + Assert.Equal(2, batch.Items.Count); + Assert.Equal(new DateTime(2018, 1, 2), batch.CommitTimeStamp); + Assert.Equal(items[0], batch.Items[0]); + Assert.Equal(items[1], batch.Items[1]); + } + + [Fact] + public async Task ReturnsEmptyItemLIsResultsInEmptyBatchList() + { + var items = new CatalogCommitItem[0]; + + var batches = await _target.CreateBatchesAsync(items); + + Assert.Empty(batches); + } + } + + public class OnProcessBatchAsync : BaseFacts + { + public OnProcessBatchAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task DoesNotFetchLeavesForDeleteEntries() + { + var items = new[] + { + new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + new CatalogCommitItem( + uri: new Uri("https://example/1"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 2), + types: null, + typeUris: new List { Schema.DataTypes.PackageDelete }, + packageIdentity: new PackageIdentity("NuGet.Frameworks", NuGetVersion.Parse("2.0.0"))), + }; + + await _target.OnProcessBatchAsync(items); + + _catalogClient.Verify(x => x.GetPackageDetailsLeafAsync("https://example/0"), Times.Once); + _catalogClient.Verify(x => x.GetPackageDetailsLeafAsync(It.IsAny()), Times.Exactly(1)); + _catalogClient.Verify(x => x.GetPackageDeleteLeafAsync(It.IsAny()), Times.Never); + + _catalogIndexActionBuilder.Verify( + x => x.AddCatalogEntriesAsync( + "NuGet.Versioning", + It.Is>( + y => y.Count == 1), + It.Is>( + y => y.Count == 1)), + Times.Once); + _catalogIndexActionBuilder.Verify( + x => x.AddCatalogEntriesAsync( + "NuGet.Frameworks", + It.Is>( + y => y.Count == 1), + It.Is>( + y => y.Count == 0)), + Times.Once); + + _batchPusher.Verify( + x => x.EnqueueIndexActions("NuGet.Versioning", It.IsAny()), + Times.Once); + _batchPusher.Verify( + x => x.EnqueueIndexActions("NuGet.Frameworks", It.IsAny()), + Times.Once); + _batchPusher.Verify( + x => x.EnqueueIndexActions(It.IsAny(), It.IsAny()), + Times.Exactly(2)); + + _batchPusher.Verify(x => x.TryFinishAsync(), Times.Once); + } + + [Fact] + public async Task OperatesOnLatestPerPackageIdentityAndGroupsById() + { + var items = new[] + { + new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + new CatalogCommitItem( + uri: new Uri("https://example/1"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 2), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + new CatalogCommitItem( + uri: new Uri("https://example/2"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 2), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("2.0.0"))), + new CatalogCommitItem( + uri: new Uri("https://example/3"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 2), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Frameworks", NuGetVersion.Parse("1.0.0"))), + }; + + await _target.OnProcessBatchAsync(items); + + _catalogClient.Verify(x => x.GetPackageDetailsLeafAsync("https://example/1"), Times.Once); + _catalogClient.Verify(x => x.GetPackageDetailsLeafAsync("https://example/2"), Times.Once); + _catalogClient.Verify(x => x.GetPackageDetailsLeafAsync("https://example/3"), Times.Once); + _catalogClient.Verify(x => x.GetPackageDetailsLeafAsync(It.IsAny()), Times.Exactly(3)); + + _catalogIndexActionBuilder.Verify( + x => x.AddCatalogEntriesAsync( + "NuGet.Versioning", + It.Is>( + y => y.Count == 2), + It.Is>( + y => y.Count == 2)), + Times.Once); + _catalogIndexActionBuilder.Verify( + x => x.AddCatalogEntriesAsync( + "NuGet.Frameworks", + It.Is>( + y => y.Count == 1), + It.Is>( + y => y.Count == 1)), + Times.Once); + + _batchPusher.Verify( + x => x.EnqueueIndexActions("NuGet.Versioning", It.IsAny()), + Times.Once); + _batchPusher.Verify( + x => x.EnqueueIndexActions("NuGet.Frameworks", It.IsAny()), + Times.Once); + _batchPusher.Verify( + x => x.EnqueueIndexActions(It.IsAny(), It.IsAny()), + Times.Exactly(2)); + + _batchPusher.Verify(x => x.TryFinishAsync(), Times.Once); + } + + [Fact] + public async Task RejectsMultipleLeavesForTheSamePackageAtTheSameTime() + { + var items = new[] + { + new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + new CatalogCommitItem( + uri: new Uri("https://example/1"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + }; + + var ex = await Assert.ThrowsAsync( + () => _target.OnProcessBatchAsync(items)); + + Assert.Equal( + "There are multiple catalog leaves for a single package at one time.", + ex.Message); + _batchPusher.Verify( + x => x.EnqueueIndexActions(It.IsAny(), It.IsAny()), + Times.Never); + } + + [Fact] + public async Task DoesNotCallFixUpEvaluatorForWhenExceptionTypeDoesNotMatch() + { + var items = new[] + { + new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + }; + + var otherException = new ArgumentException("Not so fast, buddy."); + _batchPusher + .Setup(x => x.TryFinishAsync()) + .ThrowsAsync(otherException); + + var ex = await Assert.ThrowsAsync( + () => _target.OnProcessBatchAsync(items)); + + Assert.Same(otherException, ex); + _fixUpEvaluator.Verify( + x => x.TryFixUpAsync( + It.IsAny>(), + It.IsAny>>(), + It.IsAny()), + Times.Never); + _batchPusher.Verify(x => x.TryFinishAsync(), Times.Once); + } + + [Fact] + public async Task ThrowsOriginalExceptionIfFixUpIsNotApplicable() + { + var items = new[] + { + new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + }; + + var otherException = new InvalidOperationException("Not so fast, buddy."); + _batchPusher + .Setup(x => x.TryFinishAsync()) + .ThrowsAsync(otherException); + + var ex = await Assert.ThrowsAsync( + () => _target.OnProcessBatchAsync(items)); + + Assert.Same(otherException, ex); + _fixUpEvaluator.Verify( + x => x.TryFixUpAsync( + It.IsAny>(), + It.IsAny>>(), + It.IsAny()), + Times.Once); + _batchPusher.Verify(x => x.TryFinishAsync(), Times.Once); + } + + [Fact] + public async Task ThrowsOriginalExceptionIfFixFailsThreeTimes() + { + var items = new[] + { + new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + }; + + var otherException = new InvalidOperationException("Not so fast, buddy."); + _batchPusher + .Setup(x => x.TryFinishAsync()) + .ThrowsAsync(otherException); + _fixUpEvaluator + .Setup(x => x.TryFixUpAsync( + It.IsAny>(), + It.IsAny>>(), + It.IsAny())) + .ReturnsAsync(() => DocumentFixUp.IsApplicable(new List())); + + var ex = await Assert.ThrowsAsync( + () => _target.OnProcessBatchAsync(items)); + + Assert.Same(otherException, ex); + _fixUpEvaluator.Verify( + x => x.TryFixUpAsync( + It.IsAny>(), + It.IsAny>>(), + It.IsAny()), + Times.Exactly(2)); + _batchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(3)); + } + + [Fact] + public async Task ThrowsExceptionIfPackageIdsFailThreeTimes() + { + var items = new[] + { + new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + }; + + _batchPusher + .Setup(x => x.TryFinishAsync()) + .ReturnsAsync(new BatchPusherResult(new[] { "NuGet.Versioning" })); + + var ex = await Assert.ThrowsAsync( + () => _target.OnProcessBatchAsync(items)); + + Assert.Equal("The index operations for the following package IDs failed due to version list concurrency: NuGet.Versioning", ex.Message); + _fixUpEvaluator.Verify( + x => x.TryFixUpAsync( + It.IsAny>(), + It.IsAny>>(), + It.IsAny()), + Times.Never); + _batchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(3)); + } + + [Fact] + public async Task ThrowsOriginalExceptionWithMixOfFixUpAndFailedPackageIds() + { + var items = new[] + { + new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + }; + + var otherException = new InvalidOperationException("Not so fast, buddy."); + _batchPusher + .SetupSequence(x => x.TryFinishAsync()) + .ThrowsAsync(new InvalidOperationException()) + .ReturnsAsync(new BatchPusherResult(new[] { "NuGet.Versioning" })) + .ThrowsAsync(otherException); + _fixUpEvaluator + .Setup(x => x.TryFixUpAsync( + It.IsAny>(), + It.IsAny>>(), + It.IsAny())) + .ReturnsAsync(() => DocumentFixUp.IsApplicable(new List())); + + var ex = await Assert.ThrowsAsync( + () => _target.OnProcessBatchAsync(items)); + + Assert.Same(otherException, ex); + _fixUpEvaluator.Verify( + x => x.TryFixUpAsync( + It.IsAny>(), + It.IsAny>>(), + It.IsAny()), + Times.Once); + _batchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(3)); + } + + [Fact] + public async Task RetriesOfFixUpAndFailedPackageIdsCanSucceedWithinRetries() + { + var itemsA = new[] + { + new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + }; + + var itemsB = new List + { + new CatalogCommitItem( + uri: new Uri("https://example/1"), + commitId: null, + commitTimeStamp: new DateTime(2019, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + }; + + var otherException = new InvalidOperationException("Not so fast, buddy."); + _batchPusher + .SetupSequence(x => x.TryFinishAsync()) + .ReturnsAsync(new BatchPusherResult(new[] { "NuGet.Versioning" })) + .ThrowsAsync(new InvalidOperationException()) + .ReturnsAsync(new BatchPusherResult()); + _fixUpEvaluator + .Setup(x => x.TryFixUpAsync( + It.IsAny>(), + It.IsAny>>(), + It.IsAny())) + .ReturnsAsync(() => DocumentFixUp.IsApplicable(new List(itemsB))); + + await _target.OnProcessBatchAsync(itemsA); + + _fixUpEvaluator.Verify( + x => x.TryFixUpAsync( + It.IsAny>(), + It.IsAny>>(), + It.IsAny()), + Times.Once); + _batchPusher.Verify(x => x.TryFinishAsync(), Times.Exactly(3)); + + _catalogClient.Verify(x => x.GetPackageDetailsLeafAsync("https://example/0"), Times.Exactly(2)); + _catalogClient.Verify(x => x.GetPackageDetailsLeafAsync("https://example/1"), Times.Once); + } + + [Fact] + public async Task UsesFixUpCommitItems() + { + var itemsA = new[] + { + new CatalogCommitItem( + uri: new Uri("https://example/0"), + commitId: null, + commitTimeStamp: new DateTime(2018, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + }; + + var itemsB = new List + { + new CatalogCommitItem( + uri: new Uri("https://example/1"), + commitId: null, + commitTimeStamp: new DateTime(2019, 1, 1), + types: null, + typeUris: new List { Schema.DataTypes.PackageDetails }, + packageIdentity: new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0"))), + }; + + var otherException = new InvalidOperationException("Not so fast, buddy."); + _batchPusher + .Setup(x => x.TryFinishAsync()) + .ThrowsAsync(otherException); + _fixUpEvaluator + .Setup(x => x.TryFixUpAsync( + It.IsAny>(), + It.IsAny>>(), + It.IsAny())) + .ReturnsAsync(() => DocumentFixUp.IsApplicable(itemsB)); + + await Assert.ThrowsAsync( + () => _target.OnProcessBatchAsync(itemsA)); + + _catalogClient.Verify(x => x.GetPackageDetailsLeafAsync("https://example/0"), Times.Once); + _catalogClient.Verify(x => x.GetPackageDetailsLeafAsync("https://example/1"), Times.Exactly(2)); + } + } + + public abstract class BaseFacts + { + protected readonly Mock _catalogClient; + protected readonly Mock _catalogIndexActionBuilder; + protected readonly Mock _batchPusher; + protected readonly Mock _fixUpEvaluator; + protected readonly CommitCollectorUtility _utility; + protected readonly Mock> _utilityOptions; + protected readonly Mock> _collectorOptions; + protected readonly CommitCollectorConfiguration _utilityConfig; + protected readonly Catalog2AzureSearchConfiguration _collectorConfig; + protected readonly Mock _telemetryService; + protected readonly Mock _v3TelemetryService; + protected readonly RecordingLogger _logger; + protected readonly RecordingLogger _utilityLogger; + protected readonly AzureSearchCollectorLogic _target; + + public BaseFacts(ITestOutputHelper output) + { + _catalogClient = new Mock(); + _catalogIndexActionBuilder = new Mock(); + _batchPusher = new Mock(); + _fixUpEvaluator = new Mock(); + _utilityOptions = new Mock>(); + _collectorOptions = new Mock>(); + _utilityConfig = new CommitCollectorConfiguration(); + _collectorConfig = new Catalog2AzureSearchConfiguration(); + _telemetryService = new Mock(); + _v3TelemetryService = new Mock(); + _logger = output.GetLogger(); + _utilityLogger = output.GetLogger(); + + _batchPusher.SetReturnsDefault(Task.FromResult(new BatchPusherResult())); + _utilityOptions.Setup(x => x.Value).Returns(() => _utilityConfig); + _utilityConfig.MaxConcurrentCatalogLeafDownloads = 1; + _collectorOptions.Setup(x => x.Value).Returns(() => _collectorConfig); + _collectorConfig.MaxConcurrentBatches = 1; + _fixUpEvaluator + .Setup(x => x.TryFixUpAsync( + It.IsAny>(), + It.IsAny>>(), + It.IsAny())) + .ReturnsAsync(() => DocumentFixUp.IsNotApplicable()); + + _utility = new CommitCollectorUtility( + _catalogClient.Object, + _v3TelemetryService.Object, + _utilityOptions.Object, + _utilityLogger); + + _target = new AzureSearchCollectorLogic( + _catalogIndexActionBuilder.Object, + () => _batchPusher.Object, + _fixUpEvaluator.Object, + _utility, + _collectorOptions.Object, + _telemetryService.Object, + _logger); + } + } + + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Catalog2AzureSearchCommandFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Catalog2AzureSearchCommandFacts.cs new file mode 100644 index 000000000..1e5e5e76f --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Catalog2AzureSearchCommandFacts.cs @@ -0,0 +1,158 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Moq; +using Newtonsoft.Json; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGet.Services.V3; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public class Catalog2AzureSearchCommandFacts + { + public class ExecuteAsync : BaseFacts + { + public ExecuteAsync(ITestOutputHelper output) : base(output) + { + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task ObservesCreateContainersAndIndexesOption(bool shouldCreate, bool containerExists) + { + _config.CreateContainersAndIndexes = shouldCreate; + var createIfExistsTimes = shouldCreate ? Times.Once() : Times.Never(); + var createTimes = shouldCreate && !containerExists ? Times.Once() : Times.Never(); + + await _target.ExecuteAsync(); + + _blobContainerBuilder.Verify(x => x.CreateIfNotExistsAsync(), createIfExistsTimes); + _indexBuilder.Verify(x => x.CreateSearchIndexIfNotExistsAsync(), createIfExistsTimes); + _indexBuilder.Verify(x => x.CreateHijackIndexIfNotExistsAsync(), createIfExistsTimes); + } + + [Fact] + public async Task UsesCurrentCursorValueForFront() + { + var front = new DateTime(2019, 1, 3, 0, 0, 0); + _storage.CursorValue = front; + + await _target.ExecuteAsync(); + + _collector.Verify( + x => x.RunAsync( + It.Is(c => c.Value == front), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task UsesMaxForCursorWithNoDependencies() + { + _config.DependencyCursorUrls = null; + + await _target.ExecuteAsync(); + + _collector.Verify( + x => x.RunAsync( + It.IsAny(), + It.Is(c => c.Value == DateTime.MaxValue.ToUniversalTime()), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task UsesMinOfDependencyUrlsForBack() + { + var cursor1 = "https://example/dep-cursor/1.json"; + var cursorValue1 = new DateTime(2020, 1, 1); + var cursor2 = "https://example/dep-cursor/2.json"; + var cursorValue2 = new DateTime(2020, 1, 2); + _config.DependencyCursorUrls = new List { cursor1, cursor2 }; + _httpMessageHandler + .Setup(x => x.OnSendAsync( + It.Is(m => m.RequestUri.AbsoluteUri == cursor1), + It.IsAny())) + .ReturnsAsync(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonConvert.SerializeObject(new Cursor { Value = cursorValue1 })) + }); + _httpMessageHandler + .Setup(x => x.OnSendAsync( + It.Is(m => m.RequestUri.AbsoluteUri == cursor2), + It.IsAny())) + .ReturnsAsync(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(JsonConvert.SerializeObject(new Cursor { Value = cursorValue2 })) + }); + + await _target.ExecuteAsync(); + + _collector.Verify( + x => x.RunAsync( + It.IsAny(), + It.Is(c => c.Value == cursorValue1), + It.IsAny()), + Times.Once); + } + } + + public abstract class BaseFacts + { + protected readonly Mock _collector; + protected readonly Mock _storageFactory; + protected readonly Mock _httpMessageHandler; + protected readonly Mock _blobContainerBuilder; + protected readonly Mock _indexBuilder; + protected readonly Mock> _options; + protected readonly Catalog2AzureSearchConfiguration _config; + protected readonly TestCursorStorage _storage; + protected readonly RecordingLogger _logger; + protected readonly Catalog2AzureSearchCommand _target; + + public BaseFacts(ITestOutputHelper output) + { + _collector = new Mock(); + _storageFactory = new Mock(); + _httpMessageHandler = new Mock() { CallBase = true }; + _blobContainerBuilder = new Mock(); + _indexBuilder = new Mock(); + _options = new Mock>(); + _logger = output.GetLogger(); + + _config = new Catalog2AzureSearchConfiguration + { + StorageConnectionString = "UseDevelopmentStorage=true", + StorageContainer = "container-name", + }; + _storage = new TestCursorStorage(new Uri("https://example/base/")); + + _options.Setup(x => x.Value).Returns(() => _config); + _storageFactory.Setup(x => x.Create(It.IsAny())).Returns(() => _storage); + + _target = new Catalog2AzureSearchCommand( + _collector.Object, + _storageFactory.Object, + () => _httpMessageHandler.Object, + _blobContainerBuilder.Object, + _indexBuilder.Object, + _options.Object, + _logger); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/CatalogIndexActionBuilderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/CatalogIndexActionBuilderFacts.cs new file mode 100644 index 000000000..df5a7091e --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/CatalogIndexActionBuilderFacts.cs @@ -0,0 +1,816 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using Moq; +using NuGet.Packaging.Core; +using NuGet.Protocol.Catalog; +using NuGet.Services.AzureSearch.Support; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; +using PackageDependency = NuGet.Protocol.Catalog.PackageDependency; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public class CatalogIndexActionBuilderFacts + { + public class AddCatalogEntriesAsync : BaseFacts + { + public AddCatalogEntriesAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task ThrowsWithNoEntries() + { + _latestEntries.Clear(); + + var ex = await Assert.ThrowsAsync(() => _target.AddCatalogEntriesAsync( + _packageId, + _latestEntries, + _entryToLeaf)); + Assert.Contains("There must be at least one catalog item to process.", ex.Message); + Assert.Equal("latestEntries", ex.ParamName); + } + + [Fact] + public async Task AddFirstVersion() + { + var indexActions = await _target.AddCatalogEntriesAsync( + _packageId, + _latestEntries, + _entryToLeaf); + + Assert.Equal(4, indexActions.Search.Count); + Assert.All(indexActions.Search, x => Assert.IsType(x.Document)); + Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.MergeOrUpload, x.ActionType)); + + Assert.Single(indexActions.Hijack); + Assert.IsType(indexActions.Hijack[0].Document); + Assert.Equal(IndexActionType.MergeOrUpload, indexActions.Hijack[0].ActionType); + + Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); + var properties = indexActions.VersionListDataResult.Result.VersionProperties; + Assert.Equal( + new[] { _packageVersion }, + properties.Keys.ToArray()); + Assert.True(properties[_packageVersion].Listed); + Assert.False(properties[_packageVersion].SemVer2); + + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny()), Times.Once); + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(_packageId), Times.Once); + } + + [Fact] + public async Task AddNewLatestVersion() + { + var existingVersion = "0.0.1"; + _versionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary + { + { existingVersion, new VersionPropertiesData(listed: true, semVer2: false) }, + }), + _versionListDataResult.AccessCondition); + + var indexActions = await _target.AddCatalogEntriesAsync( + _packageId, + _latestEntries, + _entryToLeaf); + + Assert.Equal(4, indexActions.Search.Count); + Assert.All(indexActions.Search, x => Assert.IsType(x.Document)); + Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.MergeOrUpload, x.ActionType)); + + Assert.Equal(2, indexActions.Hijack.Count); + var existing = indexActions.Hijack.Single(x => x.Document.Key == existingVersion); + Assert.IsType(existing.Document); + Assert.Equal(IndexActionType.Merge, existing.ActionType); + var added = indexActions.Hijack.Single(x => x.Document.Key == _packageVersion); + Assert.IsType(added.Document); + Assert.Equal(IndexActionType.MergeOrUpload, added.ActionType); + + Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); + var properties = indexActions.VersionListDataResult.Result.VersionProperties; + Assert.Equal( + new[] { existingVersion, _packageVersion }, + properties.Keys.ToArray()); + Assert.True(properties[existingVersion].Listed); + Assert.False(properties[existingVersion].SemVer2); + Assert.True(properties[_packageVersion].Listed); + Assert.False(properties[_packageVersion].SemVer2); + + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny()), Times.Once); + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(_packageId), Times.Once); + } + + [Fact] + public async Task AddNewLatestVersionForOnlySomeSearchFilters() + { + var existingVersion = "0.0.1"; + _versionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary + { + { existingVersion, new VersionPropertiesData(listed: true, semVer2: false) }, + }), + _versionListDataResult.AccessCondition); + SetVersion("1.0.0-beta"); + + var indexActions = await _target.AddCatalogEntriesAsync( + _packageId, + _latestEntries, + _entryToLeaf); + + Assert.Equal(4, indexActions.Search.Count); + var isPrerelease = indexActions.Search.ToLookup(x => x.Document.Key.Contains("Prerelease")); + Assert.All(isPrerelease[false], x => Assert.IsType(x.Document)); + Assert.All(isPrerelease[false], x => Assert.Equal(IndexActionType.Merge, x.ActionType)); + Assert.All(isPrerelease[true], x => Assert.IsType(x.Document)); + Assert.All(isPrerelease[true], x => Assert.Equal(IndexActionType.MergeOrUpload, x.ActionType)); + + Assert.Equal(2, indexActions.Hijack.Count); + var existing = indexActions.Hijack.Single(x => x.Document.Key == existingVersion); + Assert.IsType(existing.Document); + Assert.Equal(IndexActionType.Merge, existing.ActionType); + var added = indexActions.Hijack.Single(x => x.Document.Key == _packageVersion); + Assert.IsType(added.Document); + Assert.Equal(IndexActionType.MergeOrUpload, added.ActionType); + + Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); + var properties = indexActions.VersionListDataResult.Result.VersionProperties; + Assert.Equal( + new[] { existingVersion, _packageVersion }, + properties.Keys.ToArray()); + Assert.True(properties[existingVersion].Listed); + Assert.False(properties[existingVersion].SemVer2); + Assert.True(properties[_packageVersion].Listed); + Assert.False(properties[_packageVersion].SemVer2); + + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny()), Times.Once); + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(_packageId), Times.Once); + } + + [Fact] + public async Task AddNewNonLatestVersion() + { + var existingVersion = "1.0.1"; + _versionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary + { + { existingVersion, new VersionPropertiesData(listed: true, semVer2: false) }, + }), + _versionListDataResult.AccessCondition); + + var indexActions = await _target.AddCatalogEntriesAsync( + _packageId, + _latestEntries, + _entryToLeaf); + + Assert.Equal(4, indexActions.Search.Count); + Assert.All(indexActions.Search, x => Assert.IsType(x.Document)); + Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.Merge, x.ActionType)); + + Assert.Equal(2, indexActions.Hijack.Count); + var existing = indexActions.Hijack.Single(x => x.Document.Key == existingVersion); + Assert.IsType(existing.Document); + Assert.Equal(IndexActionType.Merge, existing.ActionType); + var added = indexActions.Hijack.Single(x => x.Document.Key == _packageVersion); + Assert.IsType(added.Document); + Assert.Equal(IndexActionType.MergeOrUpload, added.ActionType); + + Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); + var properties = indexActions.VersionListDataResult.Result.VersionProperties; + Assert.Equal( + new[] { _packageVersion, existingVersion }, + properties.Keys.ToArray()); + Assert.True(properties[existingVersion].Listed); + Assert.False(properties[existingVersion].SemVer2); + Assert.True(properties[_packageVersion].Listed); + Assert.False(properties[_packageVersion].SemVer2); + + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task Downgrade() + { + var existingVersion = "0.0.1"; + var existingLeaf = new PackageDetailsCatalogLeaf + { + CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero), + Url = "http://example/leaf/0.0.1", + PackageId = _packageId, + VerbatimVersion = existingVersion, + PackageVersion = existingVersion, + Listed = true, + }; + _leaf.Listed = false; + _versionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary + { + { existingVersion, new VersionPropertiesData(listed: true, semVer2: false) }, + { _packageVersion, new VersionPropertiesData(listed: true, semVer2: false) }, + }), + _versionListDataResult.AccessCondition); + _latestCatalogLeaves = new LatestCatalogLeaves( + new HashSet(), + new Dictionary + { + { NuGetVersion.Parse(existingVersion), existingLeaf }, + }); + + var indexActions = await _target.AddCatalogEntriesAsync( + _packageId, + _latestEntries, + _entryToLeaf); + + Assert.Equal(4, indexActions.Search.Count); + Assert.All(indexActions.Search, x => Assert.IsType(x.Document)); + Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.MergeOrUpload, x.ActionType)); + + Assert.Equal(2, indexActions.Hijack.Count); + var existing = indexActions.Hijack.Single(x => x.Document.Key == existingVersion); + Assert.IsType(existing.Document); + Assert.Equal(IndexActionType.MergeOrUpload, existing.ActionType); + var added = indexActions.Hijack.Single(x => x.Document.Key == _packageVersion); + Assert.IsType(added.Document); + Assert.Equal(IndexActionType.MergeOrUpload, added.ActionType); + + Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); + var properties = indexActions.VersionListDataResult.Result.VersionProperties; + Assert.Equal( + new[] { existingVersion, _packageVersion }, + properties.Keys.ToArray()); + Assert.True(properties[existingVersion].Listed); + Assert.False(properties[existingVersion].SemVer2); + Assert.False(properties[_packageVersion].Listed); + Assert.False(properties[_packageVersion].SemVer2); + + _leafFetcher.Verify( + x => x.GetLatestLeavesAsync( + It.IsAny(), + It.Is>>(y => y.Count == 1)), + Times.Once); + + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny()), Times.Once); + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(_packageId), Times.Once); + } + + [Fact] + public async Task DowngradeToDifferent() + { + var existingVersion1 = "0.0.1"; + var existingVersion2 = "0.0.2-alpha"; + var existingLeaf1 = new PackageDetailsCatalogLeaf + { + CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero), + Url = "http://example/leaf/0.0.1", + PackageId = _packageId, + VerbatimVersion = existingVersion1, + PackageVersion = existingVersion1, + Listed = true, + }; + var existingLeaf2 = new PackageDetailsCatalogLeaf + { + CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero), + Url = "http://example/leaf/0.0.2-alpha", + PackageId = _packageId, + VerbatimVersion = existingVersion2, + PackageVersion = existingVersion2, + Listed = true, + IsPrerelease = true, + }; + _leaf.Listed = false; + _versionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary + { + { existingVersion1, new VersionPropertiesData(listed: true, semVer2: false) }, + { existingVersion2, new VersionPropertiesData(listed: true, semVer2: false) }, + { _packageVersion, new VersionPropertiesData(listed: true, semVer2: false) }, + }), + _versionListDataResult.AccessCondition); + _latestCatalogLeaves = new LatestCatalogLeaves( + new HashSet(), + new Dictionary + { + { NuGetVersion.Parse(existingVersion1), existingLeaf1 }, + { NuGetVersion.Parse(existingVersion2), existingLeaf2 }, + }); + + var indexActions = await _target.AddCatalogEntriesAsync( + _packageId, + _latestEntries, + _entryToLeaf); + + Assert.Equal(4, indexActions.Search.Count); + Assert.All(indexActions.Search, x => Assert.IsType(x.Document)); + Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.MergeOrUpload, x.ActionType)); + + Assert.Equal(3, indexActions.Hijack.Count); + var existing1 = indexActions.Hijack.Single(x => x.Document.Key == existingVersion1); + Assert.IsType(existing1.Document); + Assert.Equal(IndexActionType.MergeOrUpload, existing1.ActionType); + var existing2 = indexActions.Hijack.Single(x => x.Document.Key == existingVersion2); + Assert.IsType(existing2.Document); + Assert.Equal(IndexActionType.MergeOrUpload, existing2.ActionType); + var added = indexActions.Hijack.Single(x => x.Document.Key == _packageVersion); + Assert.IsType(added.Document); + Assert.Equal(IndexActionType.MergeOrUpload, added.ActionType); + + Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); + var properties = indexActions.VersionListDataResult.Result.VersionProperties; + Assert.Equal( + new[] { existingVersion1, existingVersion2, _packageVersion }, + properties.Keys.ToArray()); + Assert.True(properties[existingVersion1].Listed); + Assert.False(properties[existingVersion1].SemVer2); + Assert.True(properties[existingVersion2].Listed); + Assert.False(properties[existingVersion2].SemVer2); + Assert.False(properties[_packageVersion].Listed); + Assert.False(properties[_packageVersion].SemVer2); + + _leafFetcher.Verify( + x => x.GetLatestLeavesAsync( + It.IsAny(), + It.Is>>(y => y.Count == 2)), + Times.Once); + + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny()), Times.Once); + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(_packageId), Times.Once); + } + + [Fact] + public async Task DowngradeToUnlist() + { + var existingVersion = "0.0.1"; + var existingLeaf = new PackageDetailsCatalogLeaf + { + CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero), + Url = "http://example/leaf/0.0.1", + PackageId = _packageId, + VerbatimVersion = existingVersion, + PackageVersion = existingVersion, + Listed = false, + }; + _leaf.Listed = false; + _versionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary + { + { existingVersion, new VersionPropertiesData(listed: true, semVer2: false) }, + { _packageVersion, new VersionPropertiesData(listed: true, semVer2: false) }, + }), + _versionListDataResult.AccessCondition); + _latestCatalogLeaves = new LatestCatalogLeaves( + new HashSet(), + new Dictionary + { + { NuGetVersion.Parse(existingVersion), existingLeaf }, + }); + + var indexActions = await _target.AddCatalogEntriesAsync( + _packageId, + _latestEntries, + _entryToLeaf); + + Assert.Equal(4, indexActions.Search.Count); + Assert.All(indexActions.Search, x => Assert.IsType(x.Document)); + Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.Delete, x.ActionType)); + + Assert.Equal(2, indexActions.Hijack.Count); + var existing = indexActions.Hijack.Single(x => x.Document.Key == existingVersion); + Assert.IsType(existing.Document); + Assert.Equal(IndexActionType.MergeOrUpload, existing.ActionType); + var added = indexActions.Hijack.Single(x => x.Document.Key == _packageVersion); + Assert.IsType(added.Document); + Assert.Equal(IndexActionType.MergeOrUpload, added.ActionType); + + Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); + var properties = indexActions.VersionListDataResult.Result.VersionProperties; + Assert.Equal( + new[] { existingVersion, _packageVersion }, + properties.Keys.ToArray()); + Assert.False(properties[existingVersion].Listed); + Assert.False(properties[existingVersion].SemVer2); + Assert.False(properties[_packageVersion].Listed); + Assert.False(properties[_packageVersion].SemVer2); + + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task DowngradeUnlistsOtherSearchFilterLatest() + { + var existingVersion1 = "2.5.11"; + var existingVersion2 = "3.0.107-pre"; + var existingVersion3 = "3.1.0+sha.8e3b68e"; + var existingLeaf1 = new PackageDetailsCatalogLeaf // This version is still listed. + { + CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero), + Url = "http://example/leaf/1", + PackageId = _packageId, + VerbatimVersion = existingVersion1, + PackageVersion = existingVersion1, + Listed = true, + }; + var existingLeaf2 = new PackageDetailsCatalogLeaf // This version is still listed. + { + CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero), + Url = "http://example/leaf/2", + PackageId = _packageId, + VerbatimVersion = existingVersion2, + PackageVersion = existingVersion2, + Listed = true, + }; + var newLeaf3 = new PackageDetailsCatalogLeaf // This version is no longer listed. + { + CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero), + Url = "http://example/leaf/3", + PackageId = _packageId, + VerbatimVersion = existingVersion3, + PackageVersion = existingVersion3, + Listed = false, + }; + + SetVersion("3.2.0-dev.1+sha.ad6878e"); + _leaf.Listed = false; + + _versionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary + { + { existingVersion1, new VersionPropertiesData(listed: true, semVer2: false) }, + { existingVersion2, new VersionPropertiesData(listed: true, semVer2: false) }, + { existingVersion3, new VersionPropertiesData(listed: true, semVer2: true) }, + { _packageVersion, new VersionPropertiesData(listed: true, semVer2: true) }, + }), + _versionListDataResult.AccessCondition); + _leafFetcher + .SetupSequence(x => x.GetLatestLeavesAsync(It.IsAny(), It.IsAny>>())) + .ReturnsAsync(new LatestCatalogLeaves( + new HashSet(), + new Dictionary + { + { NuGetVersion.Parse(existingVersion2), existingLeaf2 }, + { NuGetVersion.Parse(existingVersion3), newLeaf3 }, + })) + .ReturnsAsync(new LatestCatalogLeaves( + new HashSet(), + new Dictionary + { + { NuGetVersion.Parse(existingVersion1), existingLeaf1 }, + })) + .Throws(); + + var indexActions = await _target.AddCatalogEntriesAsync( + _packageId, + _latestEntries, + _entryToLeaf); + + Assert.Equal(4, indexActions.Search.Count); + Assert.All(indexActions.Search, x => Assert.IsType(x.Document)); + Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.MergeOrUpload, x.ActionType)); + + Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); + var properties = indexActions.VersionListDataResult.Result.VersionProperties; + Assert.Equal( + new[] { existingVersion1, existingVersion2, existingVersion3, _packageVersion }, + properties.Keys.ToArray()); + Assert.True(properties[existingVersion1].Listed); + Assert.False(properties[existingVersion1].SemVer2); + Assert.True(properties[existingVersion2].Listed); + Assert.False(properties[existingVersion2].SemVer2); + Assert.False(properties[existingVersion3].Listed); + Assert.True(properties[existingVersion3].SemVer2); + Assert.False(properties[_packageVersion].Listed); + Assert.True(properties[_packageVersion].SemVer2); + + _leafFetcher.Verify( + x => x.GetLatestLeavesAsync(_packageId, It.Is>>(y => + y.Count == 1 && + y[0].Count == 3 && + y[0][0] == NuGetVersion.Parse(existingVersion1) && + y[0][1] == NuGetVersion.Parse(existingVersion2) && + y[0][2] == NuGetVersion.Parse(existingVersion3))), + Times.Once); + _leafFetcher.Verify( + x => x.GetLatestLeavesAsync(_packageId, It.Is>>(y => + y.Count == 1 && + y[0].Count == 1 && + y[0][0] == NuGetVersion.Parse(existingVersion1))), + Times.Once); + _leafFetcher.Verify( + x => x.GetLatestLeavesAsync(It.IsAny(), It.IsAny>>()), + Times.Exactly(2)); + + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny()), Times.Once); + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(_packageId), Times.Once); + } + + [Fact] + public async Task DowngradeToDelete() + { + var existingVersion = "0.0.1"; + _leaf.Listed = false; + _versionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary + { + { existingVersion, new VersionPropertiesData(listed: true, semVer2: false) }, + { _packageVersion, new VersionPropertiesData(listed: true, semVer2: false) }, + }), + _versionListDataResult.AccessCondition); + _latestCatalogLeaves = new LatestCatalogLeaves( + new HashSet { NuGetVersion.Parse(existingVersion) }, + new Dictionary()); + + var indexActions = await _target.AddCatalogEntriesAsync( + _packageId, + _latestEntries, + _entryToLeaf); + + Assert.Equal(4, indexActions.Search.Count); + Assert.All(indexActions.Search, x => Assert.IsType(x.Document)); + Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.Delete, x.ActionType)); + + Assert.Equal(2, indexActions.Hijack.Count); + var existing = indexActions.Hijack.Single(x => x.Document.Key == existingVersion); + Assert.IsType(existing.Document); + Assert.Equal(IndexActionType.Delete, existing.ActionType); + var added = indexActions.Hijack.Single(x => x.Document.Key == _packageVersion); + Assert.IsType(added.Document); + Assert.Equal(IndexActionType.MergeOrUpload, added.ActionType); + + Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); + var properties = indexActions.VersionListDataResult.Result.VersionProperties; + Assert.Equal( + new[] { _packageVersion }, + properties.Keys.ToArray()); + Assert.False(properties[_packageVersion].Listed); + Assert.False(properties[_packageVersion].SemVer2); + + _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task DetectsSemVer2() + { + _leaf.DependencyGroups = new List + { + new PackageDependencyGroup + { + Dependencies = new List + { + new PackageDependency + { + Range = "[1.0.0-alpha.1, )", + }, + }, + }, + }; + + var indexActions = await _target.AddCatalogEntriesAsync( + _packageId, + _latestEntries, + _entryToLeaf); + + Assert.Equal(4, indexActions.Search.Count); + var isSemVer2 = indexActions.Search.ToLookup(x => x.Document.Key.Contains("SemVer2")); + Assert.All(isSemVer2[false], x => Assert.IsType(x.Document)); + Assert.All(isSemVer2[false], x => Assert.Equal(IndexActionType.Delete, x.ActionType)); + Assert.All(isSemVer2[true], x => Assert.IsType(x.Document)); + Assert.All(isSemVer2[true], x => Assert.Equal(IndexActionType.MergeOrUpload, x.ActionType)); + + Assert.Single(indexActions.Hijack); + Assert.IsType(indexActions.Hijack[0].Document); + Assert.Equal(IndexActionType.MergeOrUpload, indexActions.Hijack[0].ActionType); + + Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); + var properties = indexActions.VersionListDataResult.Result.VersionProperties; + Assert.Equal( + new[] { _packageVersion }, + properties.Keys.ToArray()); + Assert.True(properties[_packageVersion].Listed); + Assert.True(properties[_packageVersion].SemVer2); + } + + [Fact] + public async Task DetectsUnlisted() + { + _leaf.Listed = false; + + var indexActions = await _target.AddCatalogEntriesAsync( + _packageId, + _latestEntries, + _entryToLeaf); + + Assert.Equal(4, indexActions.Search.Count); + Assert.All(indexActions.Search, x => Assert.IsType(x.Document)); + Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.Delete, x.ActionType)); + + Assert.Single(indexActions.Hijack); + Assert.IsType(indexActions.Hijack[0].Document); + Assert.Equal(IndexActionType.MergeOrUpload, indexActions.Hijack[0].ActionType); + + Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); + var properties = indexActions.VersionListDataResult.Result.VersionProperties; + Assert.Equal( + new[] { _packageVersion }, + properties.Keys.ToArray()); + Assert.False(properties[_packageVersion].Listed); + Assert.False(properties[_packageVersion].SemVer2); + } + + [Fact] + public async Task AssumesDateTimeIsUtc() + { + var existingVersion = "1.0.1"; + _versionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary + { + { existingVersion, new VersionPropertiesData(listed: true, semVer2: false) }, + }), + _versionListDataResult.AccessCondition); + + await _target.AddCatalogEntriesAsync( + _packageId, + _latestEntries, + _entryToLeaf); + + _search.Verify( + x => x.UpdateVersionListFromCatalog( + It.IsAny(), + It.IsAny(), + new DateTimeOffset(_commitItem.CommitTimeStamp.Ticks, TimeSpan.Zero), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.AtLeastOnce); + + _hijack.Verify( + x => x.LatestFromCatalog( + It.IsAny(), + It.IsAny(), + new DateTimeOffset(_commitItem.CommitTimeStamp.Ticks, TimeSpan.Zero), + It.IsAny(), + It.IsAny()), + Times.AtLeastOnce); + } + } + + public abstract class BaseFacts + { + protected readonly Mock _versionListDataClient; + protected readonly Mock _leafFetcher; + protected readonly Mock _ownerFetcher; + protected readonly Mock _search; + protected readonly Mock _hijack; + protected readonly RecordingLogger _logger; + protected string _packageId; + protected string _packageVersion; + protected CatalogCommitItem _commitItem; + protected PackageDetailsCatalogLeaf _leaf; + protected readonly string[] _owners; + protected ResultAndAccessCondition _versionListDataResult; + protected List _latestEntries; + protected Dictionary _entryToLeaf; + protected LatestCatalogLeaves _latestCatalogLeaves; + protected readonly CatalogIndexActionBuilder _target; + + public BaseFacts(ITestOutputHelper output) + { + _versionListDataClient = new Mock(); + _leafFetcher = new Mock(); + _ownerFetcher = new Mock(); + _search = new Mock(); + _hijack = new Mock(); + _logger = output.GetLogger(); + + _packageId = Data.PackageId; + SetVersion("1.0.0"); + _versionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + AccessConditionWrapper.GenerateIfNotExistsCondition()); + _owners = Data.Owners; + _latestCatalogLeaves = new LatestCatalogLeaves( + new HashSet(), + new Dictionary()); + + _versionListDataClient + .Setup(x => x.ReadAsync(It.IsAny())) + .ReturnsAsync(() => _versionListDataResult); + + _search + .Setup(x => x.LatestFlagsOrNull(It.IsAny(), It.IsAny())) + .Returns((vl, sf) => new SearchDocument.LatestFlags( + vl.GetLatestVersionInfoOrNull(sf), + isLatestStable: true, + isLatest: true)); + _search + .Setup(x => x.Keyed(It.IsAny(), It.IsAny())) + .Returns( + (i, sf) => new KeyedDocument { Key = sf.ToString() }); + _search + .Setup(x => x.UpdateVersionListFromCatalog( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns( + (i, ct, ci, sf, v, ls, l) => new SearchDocument.UpdateVersionList { Key = sf.ToString() }); + _search + .Setup(x => x.UpdateVersionListAndOwnersFromCatalog( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns( + (i, ct, ci, sf, v, ls, l, o) => new SearchDocument.UpdateVersionListAndOwners { Key = sf.ToString() }); + _search + .Setup(x => x.UpdateLatestFromCatalog( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns( + (sf, v, ls, l, nv, fv, lf, o) => new SearchDocument.UpdateLatest { Key = sf.ToString() }); + + _hijack + .Setup(x => x.Keyed(It.IsAny(), It.IsAny())) + .Returns( + (i, v) => new KeyedDocument { Key = v }); + _hijack + .Setup(x => x.LatestFromCatalog( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns( + (i, v, ct, ci, c) => new HijackDocument.Latest { Key = v }); + _hijack + .Setup(x => x.FullFromCatalog(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns( + (v, c, l) => new HijackDocument.Full { Key = v }); + + _leafFetcher + .Setup(x => x.GetLatestLeavesAsync(It.IsAny(), It.IsAny>>())) + .ReturnsAsync(() => _latestCatalogLeaves); + + _ownerFetcher + .Setup(x => x.GetOwnersOrEmptyAsync(It.IsAny())) + .ReturnsAsync(() => _owners); + + _target = new CatalogIndexActionBuilder( + _versionListDataClient.Object, + _leafFetcher.Object, + _ownerFetcher.Object, + _search.Object, + _hijack.Object, + _logger); + } + + protected void SetVersion(string version) + { + var parsedVersion = NuGetVersion.Parse(version); + _packageVersion = version; + _commitItem = new CatalogCommitItem( + new Uri("https://example/uri"), + "29e5c582-c1ef-4a5c-a053-d86c7381466b", + new DateTime(2018, 11, 1), + new List { Schema.DataTypes.PackageDetails.AbsoluteUri }, + new List { Schema.DataTypes.PackageDetails }, + new PackageIdentity(_packageId, parsedVersion)); + _leaf = new PackageDetailsCatalogLeaf + { + PackageId = _packageId, + PackageVersion = _commitItem.PackageIdentity.Version.ToFullString(), + VerbatimVersion = _commitItem.PackageIdentity.Version.OriginalVersion, + IsPrerelease = parsedVersion.IsPrerelease, + Listed = true, + }; + _latestEntries = new List { _commitItem }; + _entryToLeaf = new Dictionary( + ReferenceEqualityComparer.Default) + { + { _commitItem, _leaf }, + }; + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/CatalogLeafFetcherFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/CatalogLeafFetcherFacts.cs new file mode 100644 index 000000000..a37c786cf --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/CatalogLeafFetcherFacts.cs @@ -0,0 +1,677 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Protocol.Catalog; +using NuGet.Protocol.Registration; +using NuGet.Versioning; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public class CatalogLeafFetcherFacts + { + public class GetLatestLeavesAsync : BaseFacts + { + public GetLatestLeavesAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task ConsidersAllVersionsUnavailableIfIndexIsMissing() + { + _registrationClient + .Setup(x => x.GetIndexOrNullAsync(It.IsAny())) + .ReturnsAsync((RegistrationIndex)null); + + var latest = await _target.GetLatestLeavesAsync(PackageId, _versions); + + Assert.Equal(_eachVersion, latest.Unavailable.OrderBy(x => x).ToArray()); + Assert.Empty(latest.Available); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync(It.IsAny()), Times.Once); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync("https://example/v3-registration/nuget.versioning/index.json"), Times.Once); + _registrationClient.Verify( + x => x.GetPageAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task ConsidersVersionOutsideOfRangeAsMissing() + { + var index = new RegistrationIndex + { + Items = new List + { + new RegistrationPage + { + Lower = "10.0.0", + Upper = "10.0.0", + Url = "http://example/page", + Items = new List + { + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Version = "10.0.0", + }, + }, + }, + }, + }, + }; + _registrationClient + .Setup(x => x.GetIndexOrNullAsync(It.IsAny())) + .ReturnsAsync(index); + + var latest = await _target.GetLatestLeavesAsync(PackageId, _versions); + + Assert.Equal(_eachVersion, latest.Unavailable.OrderBy(x => x).ToArray()); + Assert.Empty(latest.Available); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync(It.IsAny()), Times.Once); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync("https://example/v3-registration/nuget.versioning/index.json"), Times.Once); + _registrationClient.Verify( + x => x.GetPageAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task ConsidersMissingVersionAsUnavailable() + { + var index = new RegistrationIndex + { + Items = new List + { + new RegistrationPage + { + Lower = "0.0.0", + Upper = "10.0.0", + Url = "http://example/page", + Items = new List + { + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Version = "10.0.0", + }, + }, + }, + }, + }, + }; + _registrationClient + .Setup(x => x.GetIndexOrNullAsync(It.IsAny())) + .ReturnsAsync(index); + + var latest = await _target.GetLatestLeavesAsync(PackageId, _versions); + + Assert.Equal(_eachVersion, latest.Unavailable.OrderBy(x => x).ToArray()); + Assert.Empty(latest.Available); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync(It.IsAny()), Times.Once); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync("https://example/v3-registration/nuget.versioning/index.json"), Times.Once); + _registrationClient.Verify( + x => x.GetPageAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task ContinuesWhenVersionIsFoundToBeUnlistedOneVersionList() + { + var versions = new List> + { + new List + { + Parse("1.0.0"), + Parse("2.0.0"), + Parse("3.0.0"), + }, + }; + var details1 = new PackageDetailsCatalogLeaf { Listed = true }; + var details2 = new PackageDetailsCatalogLeaf { Listed = true }; + var details3 = new PackageDetailsCatalogLeaf { Listed = false }; + var index = new RegistrationIndex + { + Items = new List + { + new RegistrationPage + { + Lower = "1.0.0", + Upper = "3.0.0", + Url = "https://example/page", + Items = new List + { + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/1.0.0", + Version = "1.0.0", + Listed = true, + }, + }, + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/2.0.0", + Version = "2.0.0", + Listed = true, + }, + }, + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/3.0.0", + Version = "3.0.0", + Listed = false, + }, + }, + }, + }, + }, + }; + _catalogClient + .Setup(x => x.GetPackageDetailsLeafAsync("https://example/1.0.0")) + .ReturnsAsync(details1); + _catalogClient + .Setup(x => x.GetPackageDetailsLeafAsync("https://example/2.0.0")) + .ReturnsAsync(details2); + _catalogClient + .Setup(x => x.GetPackageDetailsLeafAsync("https://example/3.0.0")) + .ReturnsAsync(details3); + _registrationClient + .Setup(x => x.GetIndexOrNullAsync(It.IsAny())) + .ReturnsAsync(index); + + var latest = await _target.GetLatestLeavesAsync(PackageId, versions); + + Assert.Empty(latest.Unavailable); + Assert.Equal( + new[] { Parse("2.0.0"), Parse("3.0.0") }, + latest.Available.Keys.OrderBy(x => x).ToArray()); + Assert.Same(details2, latest.Available[Parse("2.0.0")]); + Assert.Same(details3, latest.Available[Parse("3.0.0")]); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync(It.IsAny()), Times.Once); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync("https://example/v3-registration/nuget.versioning/index.json"), Times.Once); + _registrationClient.Verify( + x => x.GetPageAsync(It.IsAny()), Times.Never); + _catalogClient.Verify( + x => x.GetPackageDetailsLeafAsync(It.IsAny()), + Times.Exactly(2)); + _catalogClient.Verify( + x => x.GetPackageDetailsLeafAsync("https://example/1.0.0"), + Times.Never); + } + + [Fact] + public async Task ContinuesWhenVersionIsFoundToBeUnlistedDifferentVersionLists() + { + var versions = new List> + { + new List + { + Parse("1.0.0+git"), + Parse("2.0.0-alpha"), + Parse("3.0.0"), + }, + new List + { + Parse("2.0.0-alpha"), + Parse("3.0.0"), + }, + }; + var details1 = new PackageDetailsCatalogLeaf { Listed = true }; + var details2 = new PackageDetailsCatalogLeaf { Listed = true }; + var details3 = new PackageDetailsCatalogLeaf { Listed = false }; + var index = new RegistrationIndex + { + Items = new List + { + new RegistrationPage + { + Lower = "1.0.0", + Upper = "3.0.0", + Url = "https://example/page", + Items = new List + { + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/1.0.0", + Version = "1.0.0+git", + Listed = true, + }, + }, + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/2.0.0-alpha", + Version = "2.0.0-alpha", + Listed = true, + }, + }, + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/3.0.0", + Version = "3.0.0", + Listed = false, + }, + }, + }, + }, + }, + }; + _catalogClient + .Setup(x => x.GetPackageDetailsLeafAsync("https://example/1.0.0")) + .ReturnsAsync(details1); + _catalogClient + .Setup(x => x.GetPackageDetailsLeafAsync("https://example/2.0.0-alpha")) + .ReturnsAsync(details2); + _catalogClient + .Setup(x => x.GetPackageDetailsLeafAsync("https://example/3.0.0")) + .ReturnsAsync(details3); + _registrationClient + .Setup(x => x.GetIndexOrNullAsync(It.IsAny())) + .ReturnsAsync(index); + + var latest = await _target.GetLatestLeavesAsync(PackageId, versions); + + Assert.Empty(latest.Unavailable); + Assert.Equal( + new[] { Parse("2.0.0-alpha"), Parse("3.0.0") }, + latest.Available.Keys.OrderBy(x => x).ToArray()); + Assert.Same(details2, latest.Available[Parse("2.0.0-alpha")]); + Assert.Same(details3, latest.Available[Parse("3.0.0")]); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync(It.IsAny()), Times.Once); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync("https://example/v3-registration/nuget.versioning/index.json"), Times.Once); + _registrationClient.Verify( + x => x.GetPageAsync(It.IsAny()), Times.Never); + _catalogClient.Verify( + x => x.GetPackageDetailsLeafAsync(It.IsAny()), + Times.Exactly(2)); + _catalogClient.Verify( + x => x.GetPackageDetailsLeafAsync("https://example/1.0.0"), + Times.Never); + } + + [Fact] + public async Task ReturnsAllProvidedVersionsIfUnlisted() + { + var versions = new List> + { + new List + { + Parse("1.0.0"), + Parse("2.0.0"), + Parse("3.0.0"), + }, + }; + var details1 = new PackageDetailsCatalogLeaf { Listed = false }; + var details2 = new PackageDetailsCatalogLeaf { Listed = false }; + var details3 = new PackageDetailsCatalogLeaf { Listed = false }; + var index = new RegistrationIndex + { + Items = new List + { + new RegistrationPage + { + Lower = "0.0.0", + Upper = "4.0.0", + Url = "https://example/page", + Items = new List + { + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/0.0.0", + Version = "0.0.0", + }, + }, + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/1.0.0", + Version = "1.0.0", + }, + }, + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/2.0.0", + Version = "2.0.0", + }, + }, + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/3.0.0", + Version = "3.0.0", + }, + }, + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/4.0.0", + Version = "4.0.0", + }, + }, + }, + }, + }, + }; + _catalogClient + .Setup(x => x.GetPackageDetailsLeafAsync("https://example/1.0.0")) + .ReturnsAsync(details1); + _catalogClient + .Setup(x => x.GetPackageDetailsLeafAsync("https://example/2.0.0")) + .ReturnsAsync(details2); + _catalogClient + .Setup(x => x.GetPackageDetailsLeafAsync("https://example/3.0.0")) + .ReturnsAsync(details3); + _registrationClient + .Setup(x => x.GetIndexOrNullAsync(It.IsAny())) + .ReturnsAsync(index); + + var latest = await _target.GetLatestLeavesAsync(PackageId, versions); + + Assert.Empty(latest.Unavailable); + Assert.Equal( + new[] { Parse("1.0.0"), Parse("2.0.0"), Parse("3.0.0") }, + latest.Available.Keys.OrderBy(x => x).ToArray()); + Assert.Same(details1, latest.Available[Parse("1.0.0")]); + Assert.Same(details2, latest.Available[Parse("2.0.0")]); + Assert.Same(details3, latest.Available[Parse("3.0.0")]); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync(It.IsAny()), Times.Once); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync("https://example/v3-registration/nuget.versioning/index.json"), Times.Once); + _registrationClient.Verify( + x => x.GetPageAsync(It.IsAny()), Times.Never); + _catalogClient.Verify( + x => x.GetPackageDetailsLeafAsync(It.IsAny()), + Times.Exactly(3)); + _catalogClient.Verify( + x => x.GetPackageDetailsLeafAsync("https://example/0.0.0"), + Times.Never); + _catalogClient.Verify( + x => x.GetPackageDetailsLeafAsync("https://example/4.0.0"), + Times.Never); + } + + [Fact] + public async Task FetchesPageIfItemsAreNotInlined() + { + var versions = new List> + { + new List + { + Parse("1.0.0"), + }, + }; + var details1 = new PackageDetailsCatalogLeaf { Listed = true }; + var page = new RegistrationPage + { + Items = new List + { + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/1.0.0", + Version = "1.0.0", + }, + }, + }, + }; + var index = new RegistrationIndex + { + Items = new List + { + new RegistrationPage + { + Lower = "1.0.0", + Upper = "1.0.0", + Url = "https://example/page", + }, + }, + }; + _catalogClient + .Setup(x => x.GetPackageDetailsLeafAsync("https://example/1.0.0")) + .ReturnsAsync(details1); + _registrationClient + .Setup(x => x.GetPageAsync("https://example/page")) + .ReturnsAsync(page); + _registrationClient + .Setup(x => x.GetIndexOrNullAsync(It.IsAny())) + .ReturnsAsync(index); + + var latest = await _target.GetLatestLeavesAsync(PackageId, versions); + + Assert.Empty(latest.Unavailable); + Assert.Equal( + new[] { Parse("1.0.0") }, + latest.Available.Keys.OrderBy(x => x).ToArray()); + Assert.Same(details1, latest.Available[Parse("1.0.0")]); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync(It.IsAny()), Times.Once); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync("https://example/v3-registration/nuget.versioning/index.json"), Times.Once); + _registrationClient.Verify( + x => x.GetPageAsync(It.IsAny()), Times.Once); + _registrationClient.Verify( + x => x.GetPageAsync("https://example/page"), Times.Once); + _catalogClient.Verify( + x => x.GetPackageDetailsLeafAsync(It.IsAny()), + Times.Once); + _catalogClient.Verify( + x => x.GetPackageDetailsLeafAsync("https://example/1.0.0"), + Times.Once); + } + + [Fact] + public async Task FetchesLeavesOnlyOnce() + { + var details1 = new PackageDetailsCatalogLeaf { Listed = true }; + var details2 = new PackageDetailsCatalogLeaf { Listed = true }; + var details3 = new PackageDetailsCatalogLeaf { Listed = true }; + var details4 = new PackageDetailsCatalogLeaf { Listed = true }; + var index = new RegistrationIndex + { + Items = new List + { + new RegistrationPage + { + Lower = "0.0.0", + Upper = "10.0.0", + Url = "https://example/page", + Items = new List + { + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/1", + Version = "1.0.0", + Listed = true, + }, + }, + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/2", + Version = "2.0.0-alpha", + Listed = true, + }, + }, + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/3", + Version = "3.0.0+git", + Listed = true, + }, + }, + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = "https://example/4", + Version = "4.0.0-beta.1", + Listed = true, + }, + }, + }, + }, + }, + }; + _catalogClient + .Setup(x => x.GetPackageDetailsLeafAsync("https://example/1")) + .ReturnsAsync(details1); + _catalogClient + .Setup(x => x.GetPackageDetailsLeafAsync("https://example/2")) + .ReturnsAsync(details2); + _catalogClient + .Setup(x => x.GetPackageDetailsLeafAsync("https://example/3")) + .ReturnsAsync(details3); + _catalogClient + .Setup(x => x.GetPackageDetailsLeafAsync("https://example/4")) + .ReturnsAsync(details4); + _registrationClient + .Setup(x => x.GetIndexOrNullAsync(It.IsAny())) + .ReturnsAsync(index); + + var latest = await _target.GetLatestLeavesAsync(PackageId, _versions); + + Assert.Empty(latest.Unavailable); + Assert.Equal( + _eachVersion, + latest.Available.Keys.OrderBy(x => x).ToArray()); + Assert.Same(details1, latest.Available[Parse("1.0.0")]); + Assert.Same(details2, latest.Available[Parse("2.0.0-alpha")]); + Assert.Same(details3, latest.Available[Parse("3.0.0")]); + Assert.Same(details4, latest.Available[Parse("4.0.0-beta.1")]); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync(It.IsAny()), Times.Once); + _registrationClient.Verify( + x => x.GetIndexOrNullAsync("https://example/v3-registration/nuget.versioning/index.json"), Times.Once); + _registrationClient.Verify( + x => x.GetPageAsync(It.IsAny()), Times.Never); + _catalogClient.Verify( + x => x.GetPackageDetailsLeafAsync(It.IsAny()), + Times.Exactly(4)); + _catalogClient.Verify( + x => x.GetPackageDetailsLeafAsync("https://example/1"), + Times.Once); + _catalogClient.Verify( + x => x.GetPackageDetailsLeafAsync("https://example/2"), + Times.Once); + _catalogClient.Verify( + x => x.GetPackageDetailsLeafAsync("https://example/3"), + Times.Once); + _catalogClient.Verify( + x => x.GetPackageDetailsLeafAsync("https://example/4"), + Times.Once); + } + } + + public abstract class BaseFacts + { + protected const string PackageId = "NuGet.Versioning"; + + protected readonly Mock _registrationClient; + protected readonly Mock _catalogClient; + protected readonly Mock> _options; + protected readonly Catalog2AzureSearchConfiguration _config; + protected readonly Mock _telemetryService; + protected readonly RecordingLogger _logger; + protected readonly List> _versions; + protected readonly NuGetVersion[] _eachVersion; + protected readonly CatalogLeafFetcher _target; + + public BaseFacts(ITestOutputHelper output) + { + _registrationClient = new Mock(); + _catalogClient = new Mock(); + _options = new Mock>(); + _config = new Catalog2AzureSearchConfiguration + { + MaxConcurrentBatches = 1, + }; + _telemetryService = new Mock(); + _logger = output.GetLogger(); + + _options.Setup(x => x.Value).Returns(() => _config); + + _config.RegistrationsBaseUrl = "https://example/v3-registration/"; + _versions = new List> + { + new List + { + Parse("1.0.0"), + }, + new List + { + Parse("1.0.0"), + Parse("2.0.0-alpha"), + }, + new List + { + Parse("1.0.0"), + Parse("3.0.0+git"), + }, + new List + { + Parse("1.0.0"), + Parse("2.0.0-alpha"), + Parse("3.0.0+git"), + Parse("4.0.0-beta.1"), + }, + }; + _eachVersion = new[] + { + Parse("1.0.0"), + Parse("2.0.0-alpha"), + Parse("3.0.0+git"), + Parse("4.0.0-beta.1"), + }; + + _target = new CatalogLeafFetcher( + _registrationClient.Object, + _catalogClient.Object, + _options.Object, + _telemetryService.Object, + _logger); + } + + protected NuGetVersion Parse(string input) + { + return NuGetVersion.Parse(input); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/DocumentFixUpEvaluatorFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/DocumentFixUpEvaluatorFacts.cs new file mode 100644 index 000000000..850c36e72 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/DocumentFixUpEvaluatorFacts.cs @@ -0,0 +1,219 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Azure.Search; +using Microsoft.Azure.Search.Models; +using Moq; +using NuGet.Packaging.Core; +using NuGet.Protocol.Catalog; +using NuGet.Services.Metadata.Catalog; +using NuGet.Versioning; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch +{ + public class DocumentFixUpEvaluatorFacts + { + public class TheTryFixUpAsyncMethod : Facts + { + public TheTryFixUpAsyncMethod(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task NoInnerExceptionIsNotApplicable() + { + var ex = new InvalidOperationException(); + + var result = await Target.TryFixUpAsync(ItemList, AllIndexActions, ex); + + Assert.False(result.Applicable); + } + + [Fact] + public async Task WrongInnerExceptionTypeIsNotApplicable() + { + var ex = new InvalidOperationException("Not good!", new ArgumentException()); + + var result = await Target.TryFixUpAsync(ItemList, AllIndexActions, ex); + + Assert.False(result.Applicable); + } + + [Fact] + public async Task No404FailureIsNotApplicable() + { + IndexingResults.Add(new IndexingResult(statusCode: 503)); + + var result = await Target.TryFixUpAsync(ItemList, AllIndexActions, Exception); + + Assert.False(result.Applicable); + } + + [Fact] + public async Task Unmatched404FailureIsNotApplicable() + { + IndexingResults.Add(new IndexingResult(statusCode: 404)); + + var result = await Target.TryFixUpAsync(ItemList, AllIndexActions, Exception); + + Assert.False(result.Applicable); + } + + [Fact] + public async Task Hijack404FailureIsNotApplicable() + { + IndexingResults.Add(new IndexingResult(key: "hijack-doc", statusCode: 404)); + AllIndexActions.Add(new IdAndValue( + "NuGet.Versioning", + new IndexActions( + search: new List>(), + hijack: new List> + { + IndexAction.Merge(new KeyedDocument { Key = "hijack-doc" }), + }, + versionListDataResult: new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + Mock.Of())))); + + var result = await Target.TryFixUpAsync(ItemList, AllIndexActions, Exception); + + Assert.False(result.Applicable); + } + + [Fact] + public async Task Search404NonMergeFailureIsNotApplicable() + { + IndexingResults.Add(new IndexingResult(key: "search-doc", statusCode: 404)); + AllIndexActions.Add(new IdAndValue( + "NuGet.Versioning", + new IndexActions( + search: new List> + { + IndexAction.Delete(new KeyedDocument { Key = "search-doc" }), + }, + hijack: new List>(), + versionListDataResult: new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + Mock.Of())))); + + var result = await Target.TryFixUpAsync(ItemList, AllIndexActions, Exception); + + Assert.False(result.Applicable); + } + + [Fact] + public async Task Search404MergeFailureIsApplicable() + { + ItemList.Add(new CatalogCommitItem( + new Uri("https://example/catalog/0.json"), + "commit-id-a", + new DateTime(2020, 3, 16, 12, 5, 0, DateTimeKind.Utc), + new string[0], + new[] { Schema.DataTypes.PackageDetails }, + new PackageIdentity("NuGet.Frameworks", NuGetVersion.Parse("1.0.0")))); + ItemList.Add(new CatalogCommitItem( + new Uri("https://example/catalog/1.json"), + "commit-id-a", + new DateTime(2020, 3, 16, 12, 5, 0, DateTimeKind.Utc), + new string[0], + new[] { Schema.DataTypes.PackageDetails }, + new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("0.9.0-beta.1")))); + + IndexingResults.Add(new IndexingResult(key: "search-doc", statusCode: 404)); + AllIndexActions.Add(new IdAndValue( + "NuGet.Versioning", + new IndexActions( + search: new List> + { + IndexAction.Merge(new KeyedDocument { Key = "search-doc" }), + }, + hijack: new List>(), + versionListDataResult: new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + Mock.Of())))); + VersionListClient + .Setup(x => x.ReadAsync(It.IsAny())) + .ReturnsAsync(() => new ResultAndAccessCondition( + new VersionListData(new Dictionary + { + { "1.0.0", new VersionPropertiesData(listed: true, semVer2: false) }, + }), + Mock.Of())); + var leaf = new PackageDetailsCatalogLeaf + { + Url = "https://example/catalog/2.json", + CommitId = "commit-id", + CommitTimestamp = new DateTimeOffset(2020, 3, 17, 12, 5, 0, TimeSpan.Zero), + Type = CatalogLeafType.PackageDetails, + }; + LeafFetcher + .Setup(x => x.GetLatestLeavesAsync( + It.IsAny(), + It.IsAny>>())) + .ReturnsAsync(() => new LatestCatalogLeaves( + new HashSet(), + new Dictionary + { + { NuGetVersion.Parse("1.0.0"), leaf }, + })); + + var result = await Target.TryFixUpAsync(ItemList, AllIndexActions, Exception); + + Assert.True(result.Applicable, "The fix up should be applicable."); + Assert.Equal(3, result.ItemList.Count); + Assert.Empty(ItemList.Except(result.ItemList)); + + var addedItem = Assert.Single(result.ItemList.Except(ItemList)); + Assert.Equal(leaf.Url, addedItem.Uri.AbsoluteUri); + Assert.Equal(leaf.CommitId, addedItem.CommitId); + Assert.Equal(leaf.CommitTimestamp, addedItem.CommitTimeStamp); + Assert.Empty(addedItem.Types); + Assert.Equal(Schema.DataTypes.PackageDetails, Assert.Single(addedItem.TypeUris)); + Assert.Equal(new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0")), addedItem.PackageIdentity); + Assert.True(addedItem.IsPackageDetails, "The generated item should be a package details item."); + Assert.False(addedItem.IsPackageDelete, "The generated item should not be a package delete item."); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + VersionListClient = new Mock(); + LeafFetcher = new Mock(); + Logger = output.GetLogger(); + + ItemList = new List(); + AllIndexActions = new ConcurrentBag>(); + IndexingResults = new List(); + DocumentIndexResult = new DocumentIndexResult(IndexingResults); + InnerException = new IndexBatchException(DocumentIndexResult); + Exception = new InvalidOperationException("It broke.", InnerException); + + Target = new DocumentFixUpEvaluator( + VersionListClient.Object, + LeafFetcher.Object, + Logger); + } + + public Mock VersionListClient { get; } + public Mock LeafFetcher { get; } + public RecordingLogger Logger { get; } + public List ItemList { get; } + public ConcurrentBag> AllIndexActions { get; } + public List IndexingResults { get; } + public DocumentIndexResult DocumentIndexResult { get; } + public IndexBatchException InnerException { get; } + public InvalidOperationException Exception { get; } + public DocumentFixUpEvaluator Target { get; } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Integration/AzureSearchCollectorLogicIntegrationTests.cs b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Integration/AzureSearchCollectorLogicIntegrationTests.cs new file mode 100644 index 000000000..5019b3fff --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Integration/AzureSearchCollectorLogicIntegrationTests.cs @@ -0,0 +1,630 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Jobs; +using NuGet.Jobs.Configuration; +using NuGet.Packaging.Core; +using NuGet.Protocol.Catalog; +using NuGet.Protocol.Registration; +using NuGet.Services.AzureSearch.Support; +using NuGet.Services.AzureSearch.Wrappers; +using NuGet.Services.Entities; +using NuGet.Services.Logging; +using NuGet.Services.Metadata.Catalog; +using NuGet.Services.V3; +using NuGet.Versioning; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch.Integration +{ + public class AzureSearchCollectorLogicIntegrationTests + { + private CommitCollectorConfiguration _utilityConfig; + private Mock> _utilityOptions; + private Catalog2AzureSearchConfiguration _config; + private AzureSearchJobDevelopmentConfiguration _developmentConfig; + private Mock> _options; + private Mock> _developmentOptions; + private Mock _telemetryClient; + private AzureSearchTelemetryService _telemetryService; + private V3TelemetryService _v3TelemetryService; + private Mock _entitiesContextFactory; + private Mock _entitiesContext; + private DatabaseAuxiliaryDataFetcher _ownerFetcher; + private InMemoryRegistrationClient _registrationClient; + private InMemoryCatalogClient _catalogClient; + private CatalogLeafFetcher _leafFetcher; + private BaseDocumentBuilder _baseDocumentBuilder; + private SearchDocumentBuilder _search; + private HijackDocumentBuilder _hijack; + private CatalogIndexActionBuilder _builder; + private InMemoryCloudBlobClient _cloudBlobClient; + private VersionListDataClient _versionListDataClient; + private Mock _searchIndex; + private InMemoryDocumentsOperations _searchDocuments; + private Mock _hijackIndex; + private InMemoryDocumentsOperations _hijackDocuments; + private DocumentFixUpEvaluator _fixUpEvaluator; + private CommitCollectorUtility _commitCollectorUtility; + private AzureSearchCollectorLogic _collector; + + public AzureSearchCollectorLogicIntegrationTests(ITestOutputHelper output) + { + _utilityConfig = new CommitCollectorConfiguration + { + MaxConcurrentCatalogLeafDownloads = 1, + }; + _utilityOptions = new Mock>(); + _utilityOptions.Setup(x => x.Value).Returns(() => _utilityConfig); + + _config = new Catalog2AzureSearchConfiguration + { + MaxConcurrentBatches = 1, + MaxConcurrentVersionListWriters = 1, + StorageContainer = "integration-tests-container", + StoragePath = "integration-tests-path", + RegistrationsBaseUrl = "https://example/registrations/", + GalleryBaseUrl = Data.GalleryBaseUrl, + FlatContainerBaseUrl = Data.FlatContainerBaseUrl, + FlatContainerContainerName = Data.FlatContainerContainerName, + + Scoring = new AzureSearchScoringConfiguration() + }; + _options = new Mock>(); + _options.Setup(x => x.Value).Returns(() => _config); + + _developmentConfig = new AzureSearchJobDevelopmentConfiguration(); + _developmentOptions = new Mock>(); + _developmentOptions.Setup(x => x.Value).Returns(() => _developmentConfig); + + _telemetryClient = new Mock(); + _telemetryService = new AzureSearchTelemetryService(_telemetryClient.Object); + _v3TelemetryService = new V3TelemetryService(_telemetryClient.Object); + + // Mock the database that is used for fetching owner information. The product code only reads + // from the database so it is less important to have a realistic, stateful implementation. + _entitiesContextFactory = new Mock(); + _entitiesContext = new Mock(); + _entitiesContextFactory.Setup(x => x.CreateAsync(It.IsAny())).ReturnsAsync(() => _entitiesContext.Object); + _entitiesContext.Setup(x => x.PackageRegistrations).Returns(DbSetMockFactory.Create()); + _ownerFetcher = new DatabaseAuxiliaryDataFetcher( + new Mock>().Object, + _entitiesContextFactory.Object, + _telemetryService, + output.GetLogger()); + + _cloudBlobClient = new InMemoryCloudBlobClient(); + _versionListDataClient = new VersionListDataClient( + _cloudBlobClient, + _options.Object, + output.GetLogger()); + _registrationClient = new InMemoryRegistrationClient(); + _catalogClient = new InMemoryCatalogClient(); + _leafFetcher = new CatalogLeafFetcher( + _registrationClient, + _catalogClient, + _options.Object, + _telemetryService, + output.GetLogger()); + _baseDocumentBuilder = new BaseDocumentBuilder(_options.Object); + _search = new SearchDocumentBuilder(_baseDocumentBuilder); + _hijack = new HijackDocumentBuilder(_baseDocumentBuilder); + _builder = new CatalogIndexActionBuilder( + _versionListDataClient, + _leafFetcher, + _ownerFetcher, + _search, + _hijack, + output.GetLogger()); + + _searchIndex = new Mock(); + _searchDocuments = new InMemoryDocumentsOperations(); + _searchIndex.Setup(x => x.Documents).Returns(() => _searchDocuments); + _hijackIndex = new Mock(); + _hijackDocuments = new InMemoryDocumentsOperations(); + _hijackIndex.Setup(x => x.Documents).Returns(() => _hijackDocuments); + + _fixUpEvaluator = new DocumentFixUpEvaluator( + _versionListDataClient, + _leafFetcher, + output.GetLogger()); + + _commitCollectorUtility = new CommitCollectorUtility( + _catalogClient, + _v3TelemetryService, + _utilityOptions.Object, + output.GetLogger()); + + _collector = new AzureSearchCollectorLogic( + _builder, + () => new BatchPusher( + _searchIndex.Object, + _hijackIndex.Object, + _versionListDataClient, + _options.Object, + _developmentOptions.Object, + _telemetryService, + output.GetLogger()), + _fixUpEvaluator, + _commitCollectorUtility, + _options.Object, + _telemetryService, + output.GetLogger()); + } + + [Fact] + public async Task AddTwoVersionsThenUnlist() + { + var identity1 = new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0.0-alpha")); + var identity2 = new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("2.0.0+git")); + + // Step #1 - add a version + { + // Arrange + var commitTimestamp = new DateTimeOffset(2018, 12, 10, 0, 0, 0, TimeSpan.Zero); + var commitId = "3998eda6-3931-4d0f-9975-d9893648d89c"; + var leafUrl = "https://example/catalog/0/nuget.versioning.1.0.0-alpha.json"; + _catalogClient.PackageDetailsLeaves[leafUrl] = CreatePackageDetailsLeaf( + commitTimestamp, + commitId, + leafUrl, + identity1, + listed: true); + var items = new[] + { + CreatePackageDetailsItem( + commitTimestamp, + commitId, + leafUrl, + identity1) + }; + + // Act + await _collector.OnProcessBatchAsync(items); + + // Assert + // Hijack documents + var hijackBatch = Assert.Single(_hijackDocuments.Batches); + var hijackAction = Assert.Single(hijackBatch.Actions); + Assert.Equal(IndexActionType.MergeOrUpload, hijackAction.ActionType); + Assert.Equal( + DocumentUtilities.GetHijackDocumentKey(identity1.Id, identity1.Version.ToNormalizedString()), + hijackAction.Document.Key); + Assert.IsType(hijackAction.Document); + + // Search documents + var searchBatch = Assert.Single(_searchDocuments.Batches); + AssertSearchBatch( + identity1.Id, + searchBatch, + exDefault: IndexActionType.Delete, + exIncludePrerelease: IndexActionType.MergeOrUpload, + exIncludeSemVer2: IndexActionType.Delete, + exIncludePrereleaseAndSemVer2: IndexActionType.MergeOrUpload); + + // Version list + var containerPair = Assert.Single(_cloudBlobClient.Containers); + Assert.Equal("integration-tests-container", containerPair.Key); + var blobPair = Assert.Single(containerPair.Value.Blobs); + Assert.Equal("integration-tests-path/version-lists/nuget.versioning.json", blobPair.Key); + Assert.Equal(@"{ + ""VersionProperties"": { + ""1.0.0-alpha"": { + ""Listed"": true + } + } +}", blobPair.Value.AsString); + } + + ClearBatches(); + + // Step #2 - add another version + { + // Arrange + var commitTimestamp = new DateTimeOffset(2018, 12, 11, 0, 0, 0, TimeSpan.Zero); + var commitId = "00c01b51-ffd4-4f55-b212-0c10d2a06dbc"; + var leafUrl = "https://example/catalog/0/nuget.versioning.2.0.0.json"; + _catalogClient.PackageDetailsLeaves[leafUrl] = CreatePackageDetailsLeaf( + commitTimestamp, + commitId, + leafUrl, + identity2, + listed: true); + var items = new[] + { + CreatePackageDetailsItem( + commitTimestamp, + commitId, + leafUrl, + identity2) + }; + + // Act + await _collector.OnProcessBatchAsync(items); + + // Assert + // Hijack documents + var hijackBatch = Assert.Single(_hijackDocuments.Batches); + var actions = hijackBatch.Actions.OrderBy(x => x.Document.Key).ToList(); + Assert.Equal(2, actions.Count); + Assert.Equal( + DocumentUtilities.GetHijackDocumentKey(identity2.Id, identity1.Version.ToNormalizedString()), + actions[0].Document.Key); + Assert.Equal(IndexActionType.Merge, actions[0].ActionType); + Assert.IsType(actions[0].Document); + Assert.Equal( + DocumentUtilities.GetHijackDocumentKey(identity2.Id, identity2.Version.ToNormalizedString()), + actions[1].Document.Key); + Assert.Equal(IndexActionType.MergeOrUpload, actions[1].ActionType); + Assert.IsType(actions[1].Document); + + // Search documents + var searchBatch = Assert.Single(_searchDocuments.Batches); + AssertSearchBatch( + identity2.Id, + searchBatch, + exDefault: IndexActionType.Delete, + exIncludePrerelease: IndexActionType.Merge, + exIncludeSemVer2: IndexActionType.MergeOrUpload, + exIncludePrereleaseAndSemVer2: IndexActionType.MergeOrUpload); + + // Version list + var containerPair = Assert.Single(_cloudBlobClient.Containers); + Assert.Equal("integration-tests-container", containerPair.Key); + var blobPair = Assert.Single(containerPair.Value.Blobs); + Assert.Equal("integration-tests-path/version-lists/nuget.versioning.json", blobPair.Key); + Assert.Equal(@"{ + ""VersionProperties"": { + ""1.0.0-alpha"": { + ""Listed"": true + }, + ""2.0.0+git"": { + ""Listed"": true, + ""SemVer2"": true + } + } +}", blobPair.Value.AsString); + } + + ClearBatches(); + + // Step #3 - unlist the first version + { + // Arrange + var commitTimestamp = new DateTimeOffset(2018, 12, 12, 0, 0, 0, TimeSpan.Zero); + var commitId = "bd9599c7-4512-4094-817e-dacef6674924"; + var leafUrl = "https://example/catalog/2/nuget.versioning.1.0.0-alpha.json"; + _catalogClient.PackageDetailsLeaves[leafUrl] = CreatePackageDetailsLeaf( + commitTimestamp, + commitId, + leafUrl, + identity1, + listed: false); + var items = new[] + { + CreatePackageDetailsItem( + commitTimestamp, + commitId, + leafUrl, + identity1), + }; + + // Act + await _collector.OnProcessBatchAsync(items); + + // Assert + // Hijack documents + var hijackBatch = Assert.Single(_hijackDocuments.Batches); + var actions = hijackBatch.Actions.OrderBy(x => x.Document.Key).ToList(); + Assert.Equal(2, actions.Count); + Assert.Equal( + DocumentUtilities.GetHijackDocumentKey(identity1.Id, identity1.Version.ToNormalizedString()), + actions[0].Document.Key); + Assert.Equal(IndexActionType.MergeOrUpload, actions[0].ActionType); + Assert.IsType(actions[0].Document); + Assert.Equal( + DocumentUtilities.GetHijackDocumentKey(identity2.Id, identity2.Version.ToNormalizedString()), + actions[1].Document.Key); + Assert.Equal(IndexActionType.Merge, actions[1].ActionType); + Assert.IsType(actions[1].Document); + + // Search documents + var searchBatch = Assert.Single(_searchDocuments.Batches); + AssertSearchBatch( + identity1.Id, + searchBatch, + exDefault: IndexActionType.Delete, + exIncludePrerelease: IndexActionType.Delete, + exIncludeSemVer2: IndexActionType.Merge, + exIncludePrereleaseAndSemVer2: IndexActionType.Merge); + + // Version list + var containerPair = Assert.Single(_cloudBlobClient.Containers); + Assert.Equal("integration-tests-container", containerPair.Key); + var blobPair = Assert.Single(containerPair.Value.Blobs); + Assert.Equal("integration-tests-path/version-lists/nuget.versioning.json", blobPair.Key); + Assert.Equal(@"{ + ""VersionProperties"": { + ""1.0.0-alpha"": {}, + ""2.0.0+git"": { + ""Listed"": true, + ""SemVer2"": true + } + } +}", blobPair.Value.AsString); + } + } + + private void ClearBatches() + { + _hijackDocuments.Clear(); + _searchDocuments.Clear(); + } + + [Fact] + public async Task DowngradeDueToDelete() + { + var identity1 = new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("1.0.0")); + var identity2 = new PackageIdentity("NuGet.Versioning", NuGetVersion.Parse("2.0.0-alpha")); + var leafUrl1 = "https://example/catalog/0/nuget.versioning.1.0.0.json"; + + // Step #1 - add two versions + { + // Arrange + var commitTimestamp = new DateTimeOffset(2018, 12, 10, 0, 0, 0, TimeSpan.Zero); + var commitId = "76d0014c-60f8-427f-8eac-3dfbf8369296"; + var leafUrl2 = "https://example/catalog/0/nuget.versioning.2.0.0-alpha.json"; + _catalogClient.PackageDetailsLeaves[leafUrl1] = CreatePackageDetailsLeaf( + commitTimestamp, + commitId, + leafUrl1, + identity1, + listed: true); + _catalogClient.PackageDetailsLeaves[leafUrl2] = CreatePackageDetailsLeaf( + commitTimestamp, + commitId, + leafUrl2, + identity2, + listed: true); + var items = new[] + { + CreatePackageDetailsItem(commitTimestamp, commitId, leafUrl1, identity1), + CreatePackageDetailsItem(commitTimestamp, commitId, leafUrl2, identity2), + }; + + // Act + await _collector.OnProcessBatchAsync(items); + + // Assert + // Hijack documents + var hijackBatch = Assert.Single(_hijackDocuments.Batches); + var actions = hijackBatch.Actions.OrderBy(x => x.Document.Key).ToList(); + Assert.Equal(2, actions.Count); + Assert.Equal( + DocumentUtilities.GetHijackDocumentKey(identity1.Id, identity1.Version.ToNormalizedString()), + actions[0].Document.Key); + Assert.Equal(IndexActionType.MergeOrUpload, actions[0].ActionType); + Assert.IsType(actions[0].Document); + Assert.Equal( + DocumentUtilities.GetHijackDocumentKey(identity2.Id, identity2.Version.ToNormalizedString()), + actions[1].Document.Key); + Assert.Equal(IndexActionType.MergeOrUpload, actions[1].ActionType); + Assert.IsType(actions[1].Document); + + // Search documents + var searchBatch = Assert.Single(_searchDocuments.Batches); + AssertSearchBatch( + identity1.Id, + searchBatch, + exDefault: IndexActionType.MergeOrUpload, + exIncludePrerelease: IndexActionType.MergeOrUpload, + exIncludeSemVer2: IndexActionType.MergeOrUpload, + exIncludePrereleaseAndSemVer2: IndexActionType.MergeOrUpload); + + // Version list + var containerPair = Assert.Single(_cloudBlobClient.Containers); + Assert.Equal("integration-tests-container", containerPair.Key); + var blobPair = Assert.Single(containerPair.Value.Blobs); + Assert.Equal("integration-tests-path/version-lists/nuget.versioning.json", blobPair.Key); + Assert.Equal(@"{ + ""VersionProperties"": { + ""1.0.0"": { + ""Listed"": true + }, + ""2.0.0-alpha"": { + ""Listed"": true + } + } +}", blobPair.Value.AsString); + } + + ClearBatches(); + + // Step #2 - delete the latest version + { + // Arrange + var commitTimestamp = new DateTimeOffset(2018, 12, 11, 0, 0, 0, TimeSpan.Zero); + var commitId = "68131297-699f-4d68-952e-c0a4eacbd6da"; + var leafUrl = "https://example/catalog/1/nuget.versioning.2.0.0-alpha.json"; + var items = new[] + { + CreatePackageDeleteItem( + commitTimestamp, + commitId, + leafUrl, + identity2), + }; + var registrationIndexUrl = "https://example/registrations/nuget.versioning/index.json"; + var registrationPageUrl = "https://example/registrations/nuget.versioning/page/0.json"; + _registrationClient.Indexes[registrationIndexUrl] = new RegistrationIndex + { + Items = new List + { + new RegistrationPage + { + Lower = identity1.Version.ToNormalizedString(), + Upper = identity1.Version.ToNormalizedString(), + Url = registrationPageUrl, + }, + }, + }; + _registrationClient.Pages[registrationPageUrl] = new RegistrationPage + { + Items = new List + { + new RegistrationLeafItem + { + CatalogEntry = new RegistrationCatalogEntry + { + Url = leafUrl1, + Version = identity1.Version.ToNormalizedString(), + } + } + } + }; + + // Act + await _collector.OnProcessBatchAsync(items); + + // Assert + // Hijack documents + var hijackBatch = Assert.Single(_hijackDocuments.Batches); + var actions = hijackBatch.Actions.OrderBy(x => x.Document.Key).ToList(); + Assert.Equal(2, actions.Count); + Assert.Equal( + DocumentUtilities.GetHijackDocumentKey(identity1.Id, identity1.Version.ToNormalizedString()), + actions[0].Document.Key); + Assert.Equal(IndexActionType.MergeOrUpload, actions[0].ActionType); + Assert.IsType(actions[0].Document); + Assert.Equal( + DocumentUtilities.GetHijackDocumentKey(identity2.Id, identity2.Version.ToNormalizedString()), + actions[1].Document.Key); + Assert.Equal(IndexActionType.Delete, actions[1].ActionType); + Assert.IsType(actions[1].Document); + + // Search documents + var searchBatch = Assert.Single(_searchDocuments.Batches); + AssertSearchBatch( + identity1.Id, + searchBatch, + exDefault: IndexActionType.MergeOrUpload, + exIncludePrerelease: IndexActionType.MergeOrUpload, + exIncludeSemVer2: IndexActionType.MergeOrUpload, + exIncludePrereleaseAndSemVer2: IndexActionType.MergeOrUpload); + + // Version list + var containerPair = Assert.Single(_cloudBlobClient.Containers); + Assert.Equal("integration-tests-container", containerPair.Key); + var blobPair = Assert.Single(containerPair.Value.Blobs); + Assert.Equal("integration-tests-path/version-lists/nuget.versioning.json", blobPair.Key); + Assert.Equal(@"{ + ""VersionProperties"": { + ""1.0.0"": { + ""Listed"": true + } + } +}", blobPair.Value.AsString); + } + } + + private static CatalogCommitItem CreatePackageDetailsItem( + DateTimeOffset commitTimestamp, + string commitId, + string leafUrl, + PackageIdentity identity) + { + return CreateItem( + commitTimestamp, + commitId, + leafUrl, + identity, + Schema.DataTypes.PackageDetails); + } + + private static CatalogCommitItem CreatePackageDeleteItem( + DateTimeOffset commitTimestamp, + string commitId, + string leafUrl, + PackageIdentity identity) + { + return CreateItem( + commitTimestamp, + commitId, + leafUrl, + identity, + Schema.DataTypes.PackageDelete); + } + + private static CatalogCommitItem CreateItem( + DateTimeOffset commitTimestamp, + string commitId, + string leafUrl, + PackageIdentity identity, + Uri type) + { + return new CatalogCommitItem( + uri: new Uri(leafUrl), + commitId: commitId, + commitTimeStamp: commitTimestamp.UtcDateTime, + types: new List(), + typeUris: new List { type }, + packageIdentity: identity); + } + + private static PackageDetailsCatalogLeaf CreatePackageDetailsLeaf( + DateTimeOffset commitTimestamp, + string commitId, + string url, + PackageIdentity identity, + bool listed) + { + return new PackageDetailsCatalogLeaf + { + CommitTimestamp = commitTimestamp, + CommitId = commitId, + Url = url, + PackageId = identity.Id, + PackageVersion = identity.Version.ToFullString(), + VerbatimVersion = identity.Version.OriginalVersion, + IsPrerelease = identity.Version.IsPrerelease, + Listed = listed, + }; + } + + private static void AssertSearchBatch( + string packageId, + IndexBatch batch, + IndexActionType exDefault, + IndexActionType exIncludePrerelease, + IndexActionType exIncludeSemVer2, + IndexActionType exIncludePrereleaseAndSemVer2) + { + Assert.Equal(4, batch.Actions.Count()); + var defaultSearch = Assert.Single( + batch.Actions, + x => DocumentUtilities.GetSearchDocumentKey(packageId, SearchFilters.Default) == x.Document.Key); + Assert.Equal(exDefault, defaultSearch.ActionType); + var includePrereleaseSearch = Assert.Single( + batch.Actions, + x => DocumentUtilities.GetSearchDocumentKey(packageId, SearchFilters.IncludePrerelease) == x.Document.Key); + Assert.Equal(exIncludePrerelease, includePrereleaseSearch.ActionType); + var includeSemVer2Search = Assert.Single( + batch.Actions, + x => DocumentUtilities.GetSearchDocumentKey(packageId, SearchFilters.IncludeSemVer2) == x.Document.Key); + Assert.Equal(exIncludeSemVer2, includeSemVer2Search.ActionType); + var includePrereleaseAndSemVer2Search = Assert.Single( + batch.Actions, + x => DocumentUtilities.GetSearchDocumentKey(packageId, SearchFilters.IncludePrereleaseAndSemVer2) == x.Document.Key); + Assert.Equal(exIncludePrereleaseAndSemVer2, includePrereleaseAndSemVer2Search.ActionType); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Integration/InMemoryDocumentsOperations.cs b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Integration/InMemoryDocumentsOperations.cs new file mode 100644 index 000000000..8b6bda615 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Catalog2AzureSearch/Integration/InMemoryDocumentsOperations.cs @@ -0,0 +1,57 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using NuGet.Services.AzureSearch.Wrappers; + +namespace NuGet.Services.AzureSearch.Catalog2AzureSearch.Integration +{ + public class InMemoryDocumentsOperations : IDocumentsOperationsWrapper + { + public ConcurrentQueue> Batches { get; } = new ConcurrentQueue>(); + + public void Clear() + { + while (Batches.Count > 0) + { + Batches.TryDequeue(out var _); + } + } + + public Task CountAsync() + { + throw new NotImplementedException(); + } + + public Task GetOrNullAsync(string key) where T : class + { + throw new NotImplementedException(); + } + + public Task IndexAsync(IndexBatch batch) where T : class + { + if (typeof(T) != typeof(KeyedDocument)) + { + throw new ArgumentException(); + } + + Batches.Enqueue(batch as IndexBatch); + + return Task.FromResult(new DocumentIndexResult(new List())); + } + + public Task> SearchAsync(string searchText, SearchParameters searchParameters) where T : class + { + throw new NotImplementedException(); + } + + public Task SearchAsync(string searchText, SearchParameters searchParameters) + { + throw new NotImplementedException(); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/DatabaseAuxiliaryDataFetcherFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/DatabaseAuxiliaryDataFetcherFacts.cs new file mode 100644 index 000000000..b169b4521 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/DatabaseAuxiliaryDataFetcherFacts.cs @@ -0,0 +1,168 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Data.Entity; +using System.Threading.Tasks; +using Moq; +using NuGet.Jobs; +using NuGet.Jobs.Configuration; +using NuGet.Services.AzureSearch.Support; +using NuGet.Services.Entities; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch +{ + public class DatabaseAuxiliaryDataFetcherFacts + { + public class GetOwnersOrEmptyAsync : Facts + { + public GetOwnersOrEmptyAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task ReturnsEmptyArrayWhenPackageRegistrationDoesNotExist() + { + var owners = await Target.GetOwnersOrEmptyAsync(Data.PackageId); + + Assert.Empty(owners); + } + + [Fact] + public async Task ReturnsEmptyArrayWhenPackageRegistrationHasNoOwners() + { + PackageRegistrations.Add(new PackageRegistration + { + Id = Data.PackageId, + Owners = new List(), + }); + + var owners = await Target.GetOwnersOrEmptyAsync(Data.PackageId); + + Assert.Empty(owners); + } + + [Fact] + public async Task ReturnsSortedOwners() + { + PackageRegistrations.Add(new PackageRegistration + { + Id = Data.PackageId, + Owners = new List + { + new User { Username = "nuget" }, + new User { Username = "aspnet" }, + new User { Username = "EntityFramework" }, + new User { Username = "Microsoft" }, + } + }); + + var owners = await Target.GetOwnersOrEmptyAsync(Data.PackageId); + + Assert.Equal(new[] { "aspnet", "EntityFramework", "Microsoft", "nuget"}, owners); + EntitiesContextFactory.Verify(x => x.CreateAsync(true), Times.Once); + EntitiesContextFactory.Verify(x => x.CreateAsync(It.IsAny()), Times.Once); + } + + [Fact] + public async Task DisposesEntitiesContext() + { + var entitiesContext = new DisposableEntitiesContext(); + EntitiesContextFactory.Setup(x => x.CreateAsync(It.IsAny())).ReturnsAsync(entitiesContext); + + var owners = await Target.GetOwnersOrEmptyAsync(Data.PackageId); + + Assert.True(entitiesContext.Disposed, "The entities context should have been disposed."); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + SqlConnectionFactory = new Mock>(); + EntitiesContextFactory = new Mock(); + EntitiesContext = new Mock(); + TelemetryService = new Mock(); + Logger = output.GetLogger(); + + PackageRegistrations = DbSetMockFactory.Create(); + + EntitiesContextFactory + .Setup(x => x.CreateAsync(It.IsAny())) + .ReturnsAsync(() => EntitiesContext.Object); + EntitiesContext + .Setup(x => x.PackageRegistrations) + .Returns(() => PackageRegistrations); + + Target = new DatabaseAuxiliaryDataFetcher( + SqlConnectionFactory.Object, + EntitiesContextFactory.Object, + TelemetryService.Object, + Logger); + } + + public Mock> SqlConnectionFactory { get; } + public Mock EntitiesContextFactory { get; } + public Mock EntitiesContext { get; } + public Mock TelemetryService { get; } + public RecordingLogger Logger { get; } + public DbSet PackageRegistrations { get; } + public DatabaseAuxiliaryDataFetcher Target { get; } + } + + private class DisposableEntitiesContext : IEntitiesContext, IDisposable + { + public DbSet Certificates { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public DbSet PackageRegistrations { get; set; } = DbSetMockFactory.Create(); + public DbSet Credentials { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public DbSet Scopes { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public DbSet Users { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public DbSet UserSecurityPolicies { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public DbSet ReservedNamespaces { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public DbSet UserCertificates { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public DbSet SymbolPackages { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public DbSet Packages { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public DbSet Deprecations { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public DbSet Vulnerabilities { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public DbSet VulnerableRanges { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public DbSet PackageRenames { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + + public bool Disposed { get; private set; } + + public void DeleteOnCommit(T entity) where T : class + { + throw new NotImplementedException(); + } + + public void Dispose() + { + Disposed = true; + } + + public IDatabase GetDatabase() + { + throw new NotImplementedException(); + } + + public Task SaveChangesAsync() + { + throw new NotImplementedException(); + } + + public DbSet Set() where T : class + { + throw new NotImplementedException(); + } + + public void SetCommandTimeout(int? seconds) + { + throw new NotImplementedException(); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/Db2AzureSearchCommandFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/Db2AzureSearchCommandFacts.cs new file mode 100644 index 000000000..416d643d7 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/Db2AzureSearchCommandFacts.cs @@ -0,0 +1,357 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Protocol.Catalog; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.Entities; +using NuGet.Services.Metadata.Catalog.Persistence; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Db2AzureSearch +{ + public class Db2AzureSearchCommandFacts + { + private readonly Mock _producer; + private readonly Mock _builder; + private readonly Mock _blobContainerBuilder; + private readonly Mock _indexBuilder; + private readonly Mock _batchPusher; + private readonly Mock _catalogClient; + private readonly Mock _storageFactory; + private readonly Mock _ownerDataClient; + private readonly Mock _downloadDataClient; + private readonly Mock _verifiedPackagesDataClient; + private readonly Mock _popularityTransferDataClient; + private readonly Mock> _options; + private readonly Mock> _developmentOptions; + private readonly Db2AzureSearchConfiguration _config; + private readonly Db2AzureSearchDevelopmentConfiguration _developmentConfig; + private readonly TestCursorStorage _storage; + private readonly InitialAuxiliaryData _initialAuxiliaryData; + private readonly RecordingLogger _logger; + private readonly Db2AzureSearchCommand _target; + + public Db2AzureSearchCommandFacts(ITestOutputHelper output) + { + _producer = new Mock(); + _builder = new Mock(); + _blobContainerBuilder = new Mock(); + _indexBuilder = new Mock(); + _batchPusher = new Mock(); + _catalogClient = new Mock(); + _storageFactory = new Mock(); + _ownerDataClient = new Mock(); + _downloadDataClient = new Mock(); + _verifiedPackagesDataClient = new Mock(); + _popularityTransferDataClient = new Mock(); + _options = new Mock>(); + _developmentOptions = new Mock>(); + _logger = output.GetLogger(); + + _config = new Db2AzureSearchConfiguration + { + MaxConcurrentBatches = 1, + StorageContainer = "container-name", + }; + _developmentConfig = new Db2AzureSearchDevelopmentConfiguration(); + _storage = new TestCursorStorage(new Uri("https://example/base/")); + _initialAuxiliaryData = new InitialAuxiliaryData( + owners: new SortedDictionary>(), + downloads: new DownloadData(), + excludedPackages: new HashSet(), + verifiedPackages: new HashSet(), + popularityTransfers: new PopularityTransferData()); + + _options + .Setup(x => x.Value) + .Returns(() => _config); + _developmentOptions + .Setup(x => x.Value) + .Returns(() => _developmentConfig); + _producer + .Setup(x => x.ProduceWorkAsync(It.IsAny>(), It.IsAny())) + .ReturnsAsync(() => _initialAuxiliaryData); + _builder + .Setup(x => x.AddNewPackageRegistration(It.IsAny())) + .Returns(() => new IndexActions( + new IndexAction[0], + new IndexAction[0], + new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + AccessConditionWrapper.GenerateEmptyCondition()))); + _batchPusher.SetReturnsDefault(Task.FromResult(new BatchPusherResult())); + _catalogClient + .Setup(x => x.GetIndexAsync(It.IsAny())) + .ReturnsAsync(new CatalogIndex()); + _storageFactory + .Setup(x => x.Create(It.IsAny())) + .Returns(() => _storage); + _blobContainerBuilder + .Setup(x => x.DeleteIfExistsAsync()) + .ReturnsAsync(true); + + _target = new Db2AzureSearchCommand( + _producer.Object, + _builder.Object, + _blobContainerBuilder.Object, + _indexBuilder.Object, + () => _batchPusher.Object, + _catalogClient.Object, + _storageFactory.Object, + _ownerDataClient.Object, + _downloadDataClient.Object, + _verifiedPackagesDataClient.Object, + _popularityTransferDataClient.Object, + _options.Object, + _developmentOptions.Object, + _logger); + } + + [Fact] + public async Task SavesCatalogCommitTimestamp() + { + var initial = new DateTimeOffset(2017, 1, 1, 12, 0, 0, TimeSpan.FromHours(4)); + _catalogClient + .Setup(x => x.GetIndexAsync(It.IsAny())) + .ReturnsAsync(new CatalogIndex { CommitTimestamp = initial }); + + await _target.ExecuteAsync(); + + Assert.Equal(new DateTime(2017, 1, 1, 8, 0, 0), _storage.CursorValue); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ObservesReplaceIndexesAndContainersOption(bool replace) + { + _developmentConfig.ReplaceContainersAndIndexes = replace; + var replaceTimes = replace ? Times.Once() : Times.Never(); + var retryOnConflict = replace; + + await _target.ExecuteAsync(); + + _blobContainerBuilder.Verify(x => x.DeleteIfExistsAsync(), replaceTimes); + _indexBuilder.Verify(x => x.DeleteSearchIndexIfExistsAsync(), replaceTimes); + _indexBuilder.Verify(x => x.DeleteHijackIndexIfExistsAsync(), replaceTimes); + _blobContainerBuilder.Verify(x => x.CreateAsync(retryOnConflict), Times.Once); + _indexBuilder.Verify(x => x.CreateSearchIndexAsync(), Times.Once); + _indexBuilder.Verify(x => x.CreateHijackIndexAsync(), Times.Once); + } + + [Fact] + public async Task PushesToIndexesUsingMaximumBatchSize() + { + _config.AzureSearchBatchSize = 2; + _producer + .Setup(x => x.ProduceWorkAsync(It.IsAny>(), It.IsAny())) + .ReturnsAsync(() => _initialAuxiliaryData) + .Callback, CancellationToken>((w, _) => + { + w.Add(new NewPackageRegistration("A", 0, new string[0], new Package[0], false)); + w.Add(new NewPackageRegistration("B", 0, new string[0], new Package[0], false)); + w.Add(new NewPackageRegistration("C", 0, new string[0], new Package[0], false)); + w.Add(new NewPackageRegistration("D", 0, new string[0], new Package[0], false)); + w.Add(new NewPackageRegistration("E", 0, new string[0], new Package[0], false)); + }); + _builder + .Setup(x => x.AddNewPackageRegistration(It.IsAny())) + .Returns(x => new IndexActions( + new List> { IndexAction.Upload(new KeyedDocument { Key = x.PackageId }) }, + new List>(), + new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + AccessConditionWrapper.GenerateEmptyCondition()))); + + var enqueuedIndexActions = new List>(); + _batchPusher + .Setup(x => x.EnqueueIndexActions(It.IsAny(), It.IsAny())) + .Callback((id, actions) => + { + enqueuedIndexActions.Add(KeyValuePair.Create(id, actions)); + }); + + await _target.ExecuteAsync(); + + Assert.Equal(5, enqueuedIndexActions.Count); + var keys = enqueuedIndexActions + .Select(x => x.Key) + .OrderBy(x => x) + .ToArray(); + Assert.Equal( + new[] { "A", "B", "C", "D", "E" }, + keys); + + _batchPusher.Verify(x => x.TryPushFullBatchesAsync(), Times.Exactly(5)); + _batchPusher.Verify(x => x.TryFinishAsync(), Times.Once); + } + + [Fact] + public async Task DoesNotEnqueueChangesForNoIndexActions() + { + _config.AzureSearchBatchSize = 2; + _producer + .Setup(x => x.ProduceWorkAsync(It.IsAny>(), It.IsAny())) + .ReturnsAsync(() => _initialAuxiliaryData) + .Callback, CancellationToken>((w, _) => + { + w.Add(new NewPackageRegistration("A", 0, new[] { "Microsoft", "EntityFramework" }, new Package[0], false)); + w.Add(new NewPackageRegistration("B", 0, new[] { "nuget" }, new Package[0], false)); + w.Add(new NewPackageRegistration("C", 0, new[] { "aspnet" }, new Package[0], false)); + }); + + // Return empty index action for ID "B". This package ID will not be pushed to Azure Search but will appear + // in the initial owners data file. + _builder + .Setup(x => x.AddNewPackageRegistration(It.Is(y => y.PackageId != "B"))) + .Returns(x => new IndexActions( + new List> { IndexAction.Upload(new KeyedDocument { Key = x.PackageId }) }, + new List>(), + new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + AccessConditionWrapper.GenerateEmptyCondition()))); + _builder + .Setup(x => x.AddNewPackageRegistration(It.Is(y => y.PackageId == "B"))) + .Returns(x => new IndexActions( + new List>(), + new List>(), + new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + AccessConditionWrapper.GenerateEmptyCondition()))); + + var enqueuedIndexActions = new List>(); + _batchPusher + .Setup(x => x.EnqueueIndexActions(It.IsAny(), It.IsAny())) + .Callback((id, actions) => + { + enqueuedIndexActions.Add(KeyValuePair.Create(id, actions)); + }); + + await _target.ExecuteAsync(); + + Assert.Equal(2, enqueuedIndexActions.Count); + var keys = enqueuedIndexActions + .Select(x => x.Key) + .OrderBy(x => x) + .ToArray(); + Assert.Equal( + new[] { "A", "C" }, + keys); + } + + [Fact] + public async Task PushesOwnerData() + { + SortedDictionary> data = null; + IAccessCondition accessCondition = null; + _ownerDataClient + .Setup(x => x.ReplaceLatestIndexedAsync(It.IsAny>>(), It.IsAny())) + .Returns(Task.CompletedTask) + .Callback>, IAccessCondition>((d, a) => + { + data = d; + accessCondition = a; + }); + + await _target.ExecuteAsync(); + + Assert.Same(_initialAuxiliaryData.Owners, data); + + Assert.Equal("*", accessCondition.IfNoneMatchETag); + Assert.Null(accessCondition.IfMatchETag); + + _ownerDataClient.Verify( + x => x.ReplaceLatestIndexedAsync(It.IsAny>>(), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task PushesDownloadData() + { + DownloadData data = null; + IAccessCondition accessCondition = null; + _downloadDataClient + .Setup(x => x.ReplaceLatestIndexedAsync(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask) + .Callback((d, a) => + { + data = d; + accessCondition = a; + }); + + await _target.ExecuteAsync(); + + Assert.Same(_initialAuxiliaryData.Downloads, data); + + Assert.Equal("*", accessCondition.IfNoneMatchETag); + Assert.Null(accessCondition.IfMatchETag); + + _downloadDataClient.Verify( + x => x.ReplaceLatestIndexedAsync(It.IsAny(), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task PushesVerifiedPackagesData() + { + HashSet data = null; + IAccessCondition accessCondition = null; + _verifiedPackagesDataClient + .Setup(x => x.ReplaceLatestAsync(It.IsAny>(), It.IsAny())) + .Returns(Task.CompletedTask) + .Callback, IAccessCondition>((d, a) => + { + data = d; + accessCondition = a; + }); + + await _target.ExecuteAsync(); + + Assert.Same(_initialAuxiliaryData.VerifiedPackages, data); + + Assert.Equal("*", accessCondition.IfNoneMatchETag); + Assert.Null(accessCondition.IfMatchETag); + + _verifiedPackagesDataClient.Verify( + x => x.ReplaceLatestAsync(It.IsAny>(), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task PushesPopularityTransferData() + { + PopularityTransferData data = null; + IAccessCondition accessCondition = null; + _popularityTransferDataClient + .Setup(x => x.ReplaceLatestIndexedAsync(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask) + .Callback((d, a) => + { + data = d; + accessCondition = a; + }); + + await _target.ExecuteAsync(); + + Assert.Same(_initialAuxiliaryData.PopularityTransfers, data); + + Assert.Equal("*", accessCondition.IfNoneMatchETag); + Assert.Null(accessCondition.IfMatchETag); + + _popularityTransferDataClient.Verify( + x => x.ReplaceLatestIndexedAsync(It.IsAny(), It.IsAny()), + Times.Once); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/EnumerableExtensionsFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/EnumerableExtensionsFacts.cs new file mode 100644 index 000000000..3d37e186e --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/EnumerableExtensionsFacts.cs @@ -0,0 +1,129 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Linq; +using Xunit; + +namespace NuGet.Services.AzureSearch.Db2AzureSearch +{ + public class EnumerableExtensionsFacts + { + public class Batch + { + [Fact] + public void PutsItemAboveMaxInOwnBatch() + { + var items = new[] + { + "aaa", + "bb", + "c", + "ddddddddddd", + "eeee", + }; + + var batches = BatchAndJoin(items, 8); + + Assert.Equal( + new[] + { + "aaa|bb|c", + "ddddddddddd", + "eeee" + }, + batches); + } + + [Fact] + public void AllowsBatchSizeToExactlyMatch() + { + var items = new[] + { + "aaa", + "bb", + "c", + "ddddddddddd", + "ee", + "ffff", + }; + + var batches = BatchAndJoin(items, 6); + + Assert.Equal( + new[] + { + "aaa|bb|c", + "ddddddddddd", + "ee|ffff" + }, + batches); + } + + [Fact] + public void HandlesZeroItems() + { + var items = new[] + { + "aaa", + "", + "", + "b", + "ccccc", + "", + "", + "", + "d" + }; + + var batches = BatchAndJoin(items, 5); + + Assert.Equal( + new[] + { + "aaa|||b", + "ccccc|||", + "d" + }, + batches); + } + + [Fact] + public void ReturnsEmptyForEmpty() + { + var items = new string[0]; + + var batches = BatchAndJoin(items, 8); + + Assert.Empty(batches); + } + + [Fact] + public void AllowsSingleElementSmallerThanMax() + { + var items = new[] { "a" }; + + var batches = BatchAndJoin(items, 8); + + Assert.Equal(new[] { "a" }, batches); + } + + [Fact] + public void AllowsSingleElementLargerThanMax() + { + var items = new[] { "aaa" }; + + var batches = BatchAndJoin(items, 2); + + Assert.Equal(new[] { "aaa" }, batches); + } + + private static string[] BatchAndJoin(string[] items, int maxSize) + { + return items + .Batch(x => x.Length, maxSize) + .Select(x => string.Join("|", x)) + .ToArray(); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/NewPackageRegistrationProducerFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/NewPackageRegistrationProducerFacts.cs new file mode 100644 index 000000000..4d523e265 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/NewPackageRegistrationProducerFacts.cs @@ -0,0 +1,698 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Data.Entity; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Moq; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.Entities; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Db2AzureSearch +{ + public class NewPackageRegistrationProducerFacts + { + public class ProduceWorkAsync + { + private readonly Mock _entitiesContextFactory; + private readonly Mock _entitiesContext; + private readonly Mock> _options; + private readonly Mock> _developmentOptions; + private readonly Db2AzureSearchConfiguration _config; + private readonly Db2AzureSearchDevelopmentConfiguration _developmentConfig; + private readonly RecordingLogger _logger; + private readonly DbSet _packageRegistrations; + private readonly DbSet _packages; + private readonly ConcurrentBag _work; + private readonly CancellationToken _token; + private readonly NewPackageRegistrationProducer _target; + private readonly Mock _auxiliaryFileClient; + private readonly Mock _databaseFetcher; + private readonly Mock _downloadTransferrer; + private readonly Mock _featureFlags; + private readonly DownloadData _downloads; + private readonly PopularityTransferData _popularityTransfers; + private readonly SortedDictionary _transferChanges; + private HashSet _excludedPackages; + + public ProduceWorkAsync(ITestOutputHelper output) + { + _entitiesContextFactory = new Mock(); + _entitiesContext = new Mock(); + _options = new Mock>(); + _config = new Db2AzureSearchConfiguration + { + DatabaseBatchSize = 2, + EnablePopularityTransfers = true, + }; + _developmentOptions = new Mock>(); + _developmentConfig =new Db2AzureSearchDevelopmentConfiguration(); + _logger = output.GetLogger(); + _packageRegistrations = DbSetMockFactory.Create(); + _packages = DbSetMockFactory.Create(); + _work = new ConcurrentBag(); + _token = CancellationToken.None; + + _auxiliaryFileClient = new Mock(); + _excludedPackages = new HashSet(StringComparer.OrdinalIgnoreCase); + _auxiliaryFileClient + .Setup(x => x.LoadExcludedPackagesAsync()) + .ReturnsAsync(() => _excludedPackages); + _downloads = new DownloadData(); + _auxiliaryFileClient + .Setup(x => x.LoadDownloadDataAsync()) + .ReturnsAsync(() => _downloads); + + _popularityTransfers = new PopularityTransferData(); + _databaseFetcher = new Mock(); + _databaseFetcher + .Setup(x => x.GetPopularityTransfersAsync()) + .ReturnsAsync(() => _popularityTransfers); + + _downloadTransferrer = new Mock(); + _transferChanges = new SortedDictionary(StringComparer.OrdinalIgnoreCase); + _downloadTransferrer + .Setup(x => x.InitializeDownloadTransfers( + It.IsAny(), + It.IsAny())) + .Returns(_transferChanges); + + _featureFlags = new Mock(); + _featureFlags + .Setup(x => x.IsPopularityTransferEnabled()) + .Returns(true); + + _entitiesContextFactory + .Setup(x => x.CreateAsync(It.IsAny())) + .ReturnsAsync(() => _entitiesContext.Object); + _entitiesContext + .Setup(x => x.Set()) + .Returns(() => _packageRegistrations); + _entitiesContext + .Setup(x => x.Set()) + .Returns(() => _packages); + _options + .Setup(x => x.Value) + .Returns(() => _config); + _developmentOptions + .Setup(x => x.Value) + .Returns(() => _developmentConfig); + + _target = new NewPackageRegistrationProducer( + _entitiesContextFactory.Object, + _auxiliaryFileClient.Object, + _databaseFetcher.Object, + _downloadTransferrer.Object, + _featureFlags.Object, + _options.Object, + _developmentOptions.Object, + _logger); + } + + [Fact] + public async Task AllowsNoWork() + { + await _target.ProduceWorkAsync(_work, _token); + + Assert.Empty(_work); + _entitiesContextFactory.Verify(x => x.CreateAsync(true), Times.Once); + _entitiesContext.Verify(x => x.Set(), Times.Once); + _entitiesContext.Verify(x => x.Set(), Times.Never); + } + + [Fact] + public async Task DoesNotCountUnavailablePackages() + { + _packageRegistrations.Add(new PackageRegistration + { + Key = 1, + Id = "A", + Packages = new[] + { + new Package { Version = "1.0.0", PackageStatusKey = PackageStatus.Deleted }, + new Package { Version = "2.0.0" }, + } + }); + _packageRegistrations.Add(new PackageRegistration + { + Key = 2, + Id = "B", + Packages = new[] + { + new Package { Version = "3.0.0" }, + } + }); + _packageRegistrations.Add(new PackageRegistration + { + Key = 3, + Id = "C", + Packages = new[] + { + new Package { Version = "4.0.0" }, + } + }); + InitializePackagesFromPackageRegistrations(); + + await _target.ProduceWorkAsync(_work, _token); + + Assert.Equal(3, _work.Count); + Assert.Contains( + "Fetching packages with package registration key >= 1 and <= 2 (~2 packages).", + _logger.Messages); + Assert.Contains( + "Fetching packages with package registration key >= 3 (~1 packages).", + _logger.Messages); + } + + [Fact] + public async Task FillsGapsInRanges() + { + _packageRegistrations.Add(new PackageRegistration + { + Key = 2, + Id = "A", + Packages = new[] + { + new Package { Version = "1.0.0" }, + } + }); + _packageRegistrations.Add(new PackageRegistration + { + Key = 10, + Id = "B", + Packages = new[] + { + new Package { Version = "2.0.0" }, + } + }); + _packageRegistrations.Add(new PackageRegistration + { + Key = 1000, + Id = "C", + Packages = new[] + { + new Package { Version = "3.0.0" }, + } + }); + InitializePackagesFromPackageRegistrations(); + + await _target.ProduceWorkAsync(_work, _token); + + Assert.Equal(3, _work.Count); + Assert.Contains( + "Fetching packages with package registration key >= 1 and <= 999 (~2 packages).", + _logger.Messages); + Assert.Contains( + "Fetching packages with package registration key >= 1000 (~1 packages).", + _logger.Messages); + } + + [Fact] + public async Task ProducesWorkPerPackageRegistration() + { + _config.DatabaseBatchSize = 4; + _packageRegistrations.Add(new PackageRegistration + { + Key = 1, + Id = "A", + Owners = new[] { new User { Username = "OwnerA" } }, + Packages = new[] + { + new Package { Version = "1.0.0" }, + new Package { Version = "2.0.0" }, + }, + }); + _downloads.SetDownloadCount("A", "1.0.0", 23); + _packageRegistrations.Add(new PackageRegistration + { + Key = 2, + Id = "B", + Owners = new[] { new User { Username = "OwnerB" } }, + Packages = new[] + { + new Package { Version = "3.0.0" }, + }, + }); + _downloads.SetDownloadCount("B", "3.0.0", 24); + _packageRegistrations.Add(new PackageRegistration + { + Key = 3, + Id = "C", + Owners = new[] { new User { Username = "OwnerC" }, new User { Username = "OwnerD" } }, + Packages = new[] + { + new Package { Version = "4.0.0" }, + }, + }); + _downloads.SetDownloadCount("C", "4.0.0", 25); + _packageRegistrations.Add(new PackageRegistration + { + Key = 4, + Id = "D", + DownloadCount = 26, + Owners = new[] { new User { Username = "OwnerE" } }, + Packages = new Package[0], + }); + _downloads.SetDownloadCount("D", "5.0.0", 26); + InitializePackagesFromPackageRegistrations(); + + await _target.ProduceWorkAsync(_work, _token); + + var work = _work.Reverse().ToList(); + Assert.Equal(4, work.Count); + + Assert.Equal("A", work[0].PackageId); + Assert.Equal("1.0.0", work[0].Packages[0].Version); + Assert.Equal("2.0.0", work[0].Packages[1].Version); + Assert.Equal(new[] { "OwnerA" }, work[0].Owners); + Assert.Equal(23, work[0].TotalDownloadCount); + + Assert.Equal("B", work[1].PackageId); + Assert.Equal("3.0.0", work[1].Packages[0].Version); + Assert.Equal(new[] { "OwnerB" }, work[1].Owners); + Assert.Equal(24, work[1].TotalDownloadCount); + + Assert.Equal("C", work[2].PackageId); + Assert.Equal("4.0.0", work[2].Packages[0].Version); + Assert.Equal(new[] { "OwnerC", "OwnerD" }, work[2].Owners); + Assert.Equal(25, work[2].TotalDownloadCount); + + Assert.Equal("D", work[3].PackageId); + Assert.Empty(work[3].Packages); + Assert.Equal(new[] { "OwnerE" }, work[3].Owners); + Assert.Equal(26, work[3].TotalDownloadCount); + } + + [Fact] + public async Task DefaultsPackageDownloads() + { + _packageRegistrations.Add(new PackageRegistration + { + Key = 1, + Id = "HasDownloads", + Packages = new[] + { + new Package { Version = "1.0.0" }, + }, + }); + _downloads.SetDownloadCount("HasDownloads", "1.0.0", 100); + _packageRegistrations.Add(new PackageRegistration + { + Key = 2, + Id = "NoDownloads", + Packages = new[] + { + new Package { Version = "1.0.0" }, + }, + }); + + InitializePackagesFromPackageRegistrations(); + + var result = await _target.ProduceWorkAsync(_work, _token); + + // Documents should have overriden downloads. + var work = _work.Reverse().ToList(); + Assert.Equal(2, work.Count); + + Assert.Equal("HasDownloads", work[0].PackageId); + Assert.Equal("1.0.0", work[0].Packages[0].Version); + Assert.Equal(100, work[0].TotalDownloadCount); + Assert.Equal("NoDownloads", work[1].PackageId); + Assert.Equal("1.0.0", work[1].Packages[0].Version); + Assert.Equal(0, work[1].TotalDownloadCount); + + // Downloads auxiliary file should have original downloads. + Assert.Equal(100, result.Downloads["HasDownloads"]["1.0.0"]); + Assert.False(result.Downloads.ContainsKey("NoDownloads")); + } + + [Fact] + public async Task RetrievesAndUsesExclusionList() + { + _packageRegistrations.Add(new PackageRegistration + { + Key = 1, + Id = "A", + Owners = new User[0], + Packages = new[] + { + new Package { Version = "1.0.0" }, + new Package { Version = "2.0.0" }, + }, + }); + _packageRegistrations.Add(new PackageRegistration + { + Key = 2, + Id = "B", + Owners = new User[0], + Packages = new[] + { + new Package { Version = "3.0.0" }, + }, + }); + _packageRegistrations.Add(new PackageRegistration + { + Key = 3, + Id = "C", + Owners = new User[0], + Packages = new[] + { + new Package { Version = "4.0.0" }, + }, + }); + _packageRegistrations.Add(new PackageRegistration + { + Key = 4, + Id = "D", + Owners = new User[0], + Packages = new Package[0], + }); + + InitializePackagesFromPackageRegistrations(); + + _excludedPackages = new HashSet(StringComparer.OrdinalIgnoreCase) { "A", "C" }; + + await _target.ProduceWorkAsync(_work, _token); + + var work = _work.Reverse().ToList(); + Assert.Equal(4, work.Count); + for (int i = 0; i < work.Count; i++) + { + var shouldBeExcluded = _excludedPackages.Contains(work[i].PackageId, StringComparer.OrdinalIgnoreCase); + Assert.Equal(shouldBeExcluded, work[i].IsExcludedByDefault); + } + } + + [Fact] + public async Task SkipsUnwantedPackages() + { + _developmentConfig.SkipPackagePrefixes = new List { "Foo" }; + + _packageRegistrations.Add(new PackageRegistration + { + Key = 1, + Id = "FOO.Bar", + Owners = new User[0], + Packages = new[] + { + new Package { Version = "1.0.0" }, + }, + }); + _packageRegistrations.Add(new PackageRegistration + { + Key = 2, + Id = "foo.Buzz", + Owners = new User[0], + Packages = new[] + { + new Package { Version = "2.0.0" }, + }, + }); + _packageRegistrations.Add(new PackageRegistration + { + Key = 3, + Id = "Hello.World", + Owners = new User[0], + Packages = new[] + { + new Package { Version = "3.0.0" }, + }, + }); + + InitializePackagesFromPackageRegistrations(); + + await _target.ProduceWorkAsync(_work, _token); + + var newRegistration = Assert.Single(_work); + Assert.Equal("Hello.World", newRegistration.PackageId); + } + + [Fact] + public async Task ReturnsInitialAuxiliaryData() + { + _packageRegistrations.Add(new PackageRegistration + { + Key = 1, + Id = "A", + Owners = new[] { new User { Username = "OwnerA" } }, + Packages = new[] + { + new Package { Version = "1.0.0" }, + new Package { Version = "2.0.0" }, + }, + IsVerified = true, + }); + + var output = await _target.ProduceWorkAsync(_work, _token); + + Assert.Same(_downloads, output.Downloads); + Assert.Same(_excludedPackages, output.ExcludedPackages); + Assert.Same(_popularityTransfers, output.PopularityTransfers); + Assert.NotNull(output.VerifiedPackages); + Assert.Contains("A", output.VerifiedPackages); + Assert.NotNull(output.Owners); + Assert.Contains("A", output.Owners.Keys); + Assert.Equal(new[] { "OwnerA" }, output.Owners["A"].ToArray()); + + _auxiliaryFileClient.Verify(x => x.LoadExcludedPackagesAsync(), Times.Once); + } + + [Fact] + public async Task ThrowsWhenExcludedPackagesIsMissing() + { + _auxiliaryFileClient + .Setup(x => x.LoadExcludedPackagesAsync()) + .ThrowsAsync(new StorageException( + new RequestResult + { + HttpStatusCode = (int)HttpStatusCode.NotFound, + }, + message: "Not found.", + inner: null)); + + await Assert.ThrowsAsync(async () => await _target.ProduceWorkAsync(_work, _token)); + } + + [Fact] + public async Task AppliesDownloadTransfers() + { + _packageRegistrations.Add(new PackageRegistration + { + Key = 1, + Id = "A", + Packages = new[] + { + new Package { Version = "1.0.0" }, + new Package { Version = "2.0.0" }, + }, + }); + _downloads.SetDownloadCount("A", "1.0.0", 12); + _downloads.SetDownloadCount("A", "2.0.0", 23); + _packageRegistrations.Add(new PackageRegistration + { + Key = 2, + Id = "B", + Packages = new[] + { + new Package { Version = "3.0.0" }, + new Package { Version = "4.0.0" }, + }, + }); + _downloads.SetDownloadCount("B", "3.0.0", 5); + _downloads.SetDownloadCount("B", "4.0.0", 4); + _packageRegistrations.Add(new PackageRegistration + { + Key = 3, + Id = "C", + Packages = new[] + { + new Package { Version = "5.0.0" }, + new Package { Version = "6.0.0" }, + }, + }); + _downloads.SetDownloadCount("C", "5.0.0", 2); + _downloads.SetDownloadCount("C", "6.0.0", 3); + + InitializePackagesFromPackageRegistrations(); + + // Transfer changes should be applied to the package registrations. + _transferChanges["A"] = 55; + _transferChanges["b"] = 66; + _transferChanges["C"] = 123; + + var result = await _target.ProduceWorkAsync(_work, _token); + _databaseFetcher.Verify(x => x.GetPopularityTransfersAsync(), Times.Once); + + _downloadTransferrer + .Verify( + x => x.InitializeDownloadTransfers(_downloads, _popularityTransfers), + Times.Once); + + // Documents should have overriden downloads. + var work = _work.Reverse().ToList(); + Assert.Equal(3, work.Count); + + Assert.Equal("A", work[0].PackageId); + Assert.Equal("1.0.0", work[0].Packages[0].Version); + Assert.Equal("2.0.0", work[0].Packages[1].Version); + Assert.Equal(55, work[0].TotalDownloadCount); + + Assert.Equal("B", work[1].PackageId); + Assert.Equal("3.0.0", work[1].Packages[0].Version); + Assert.Equal("4.0.0", work[1].Packages[1].Version); + Assert.Equal(66, work[1].TotalDownloadCount); + + Assert.Equal("C", work[2].PackageId); + Assert.Equal("5.0.0", work[2].Packages[0].Version); + Assert.Equal("6.0.0", work[2].Packages[1].Version); + Assert.Equal(123, work[2].TotalDownloadCount); + + // Downloads auxiliary file should have original downloads. + Assert.Equal(12, result.Downloads["A"]["1.0.0"]); + Assert.Equal(23, result.Downloads["A"]["2.0.0"]); + Assert.Equal(5, result.Downloads["B"]["3.0.0"]); + Assert.Equal(4, result.Downloads["B"]["4.0.0"]); + Assert.Equal(2, result.Downloads["C"]["5.0.0"]); + Assert.Equal(3, result.Downloads["C"]["6.0.0"]); + } + + [Fact] + public async Task ConfigDisablesPopularityTransfers() + { + _packageRegistrations.Add(new PackageRegistration + { + Key = 1, + Id = "A", + Packages = new[] + { + new Package { Version = "1.0.0" }, + }, + }); + _downloads.SetDownloadCount("A", "1.0.0", 100); + _popularityTransfers.AddTransfer("A", "A"); + + InitializePackagesFromPackageRegistrations(); + + _config.EnablePopularityTransfers = false; + + var result = await _target.ProduceWorkAsync(_work, _token); + + // The popularity transfers should not be loaded from the database. + _databaseFetcher + .Verify( + x => x.GetPopularityTransfersAsync(), + Times.Never); + + // Popularity transfers should not be passed to the download transferrer. + _downloadTransferrer + .Verify( + x => x.InitializeDownloadTransfers( + _downloads, + It.Is(data => data.Count == 0)), + Times.Once); + + // There should be no popularity transfers. + Assert.Empty(result.PopularityTransfers); + } + + [Fact] + public async Task FlagDisablesPopularityTransfers() + { + _packageRegistrations.Add(new PackageRegistration + { + Key = 1, + Id = "A", + Packages = new[] + { + new Package { Version = "1.0.0" }, + }, + }); + _downloads.SetDownloadCount("A", "1.0.0", 100); + _popularityTransfers.AddTransfer("A", "A"); + + InitializePackagesFromPackageRegistrations(); + + _featureFlags + .Setup(x => x.IsPopularityTransferEnabled()) + .Returns(false); + + var result = await _target.ProduceWorkAsync(_work, _token); + + // The popularity transfers should not be loaded from the database. + _databaseFetcher + .Verify( + x => x.GetPopularityTransfersAsync(), + Times.Never); + + // Popularity transfers should not be passed to the download transferrer. + _downloadTransferrer + .Verify( + x => x.InitializeDownloadTransfers( + _downloads, + It.Is(data => data.Count == 0)), + Times.Once); + + // There should be no popularity transfers. + Assert.Empty(result.PopularityTransfers); + } + + [Fact] + public async Task IgnoresDownloadTransfersForNonexistentPackages() + { + _packageRegistrations.Add(new PackageRegistration + { + Key = 1, + Id = "A", + Packages = new[] + { + new Package { Version = "1.0.0" }, + }, + }); + _downloads.SetDownloadCount("A", "1.0.0", 100); + + InitializePackagesFromPackageRegistrations(); + + // Transfer changes should be applied to the package registrations. + // Transfer changes for packages that do not exist should be ignored. + _transferChanges["PackageDoesNotExist"] = 123; + + var result = await _target.ProduceWorkAsync(_work, _token); + + // Documents should have overriden downloads. + var work = _work.ToList(); + Assert.Single(work); + + Assert.Equal("A", work[0].PackageId); + Assert.Equal("1.0.0", work[0].Packages[0].Version); + Assert.Equal(100, work[0].TotalDownloadCount); + + // Downloads auxiliary file should have original downloads. + Assert.Equal(100, result.Downloads["A"]["1.0.0"]); + } + + private void InitializePackagesFromPackageRegistrations() + { + foreach (var pr in _packageRegistrations) + { + foreach (var package in pr.Packages) + { + package.PackageRegistration = pr; + package.PackageRegistrationKey = pr.Key; + + _packages.Add(package); + } + } + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/PackageEntityIndexActionBuilderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/PackageEntityIndexActionBuilderFacts.cs new file mode 100644 index 000000000..37cd523c7 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Db2AzureSearch/PackageEntityIndexActionBuilderFacts.cs @@ -0,0 +1,229 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Azure.Search.Models; +using Moq; +using NuGet.Services.Entities; +using NuGet.Versioning; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Db2AzureSearch +{ + public class PackageEntityIndexActionBuilderFacts + { + public class AddNewPackageRegistration : BaseFacts + { + public AddNewPackageRegistration(ITestOutputHelper output) : base(output) + { + } + + [Theory] + [InlineData("1.0.0.0-ALPHA", "1.0.0-ALPHA")] + [InlineData("01.0.0-ALPHA", "1.0.0-ALPHA")] + [InlineData("1.0.0-ALPHA+githash", "1.0.0-ALPHA+githash")] + [InlineData("1.0.0-ALPHA.1+githash", "1.0.0-ALPHA.1+githash")] + public void UsesProperVersionForBuilder(string version, string fullVersion) + { + var input = new NewPackageRegistration( + "NuGet.Versioning", + 1001, + new string[0], + new[] { new TestPackage(version) { SemVerLevelKey = SemVerLevelKey.SemVer2 } }, + false); + + var actions = _target.AddNewPackageRegistration(input); + + _search.Verify( + x => x.Keyed(input.PackageId, SearchFilters.Default), + Times.Once); + _search.Verify( + x => x.Keyed(input.PackageId, SearchFilters.IncludePrerelease), + Times.Once); + _search.Verify( + x => x.Keyed(input.PackageId, SearchFilters.IncludeSemVer2), + Times.Once); + _search.Verify( + x => x.FullFromDb( + input.PackageId, + SearchFilters.IncludePrereleaseAndSemVer2, + It.IsAny(), + It.IsAny(), + It.IsAny(), + fullVersion, + input.Packages[0], + input.Owners, + input.TotalDownloadCount, + It.IsAny()), + Times.Once); + } + + [Fact] + public void UsesLatestVersionMetadataForSearchIndex() + { + _search + .Setup(x => x.FullFromDb( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns( + (i, sf, v, ls, l, fv, p, o, d, ie) => new SearchDocument.Full { OriginalVersion = p.Version }); + var package1 = new TestPackage("1.0.0") { Description = "This is version 1.0.0." }; + var package2 = new TestPackage("2.0.0-alpha") { Description = "This is version 2.0.0." }; + var input = new NewPackageRegistration( + "NuGet.Versioning", + 1001, + new string[0], + new[] { package1, package2 }, + false); + + var actions = _target.AddNewPackageRegistration(input); + + Assert.Equal(4, actions.Search.Count); + Assert.Equal(IndexActionType.Upload, actions.Search[0].ActionType); // SearchFilters.Default + Assert.Equal(IndexActionType.Upload, actions.Search[1].ActionType); // SearchFilters.IncludePrerelease + Assert.Equal(IndexActionType.Upload, actions.Search[2].ActionType); // SearchFilters.IncludeSemVer2 + Assert.Equal(IndexActionType.Upload, actions.Search[3].ActionType); // SearchFilters.IncludePrereleaseAndSemVer2 + var doc0 = Assert.IsType(actions.Search[0].Document); + Assert.Equal(package1.Version, doc0.OriginalVersion); + var doc1 = Assert.IsType(actions.Search[1].Document); + Assert.Equal(package2.Version, doc1.OriginalVersion); + var doc2 = Assert.IsType(actions.Search[2].Document); + Assert.Equal(package1.Version, doc2.OriginalVersion); + var doc3 = Assert.IsType(actions.Search[3].Document); + Assert.Equal(package2.Version, doc3.OriginalVersion); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void PassesIsExcludedByDefaultValueCorrectly(bool shouldBeExcluded) + { + var input = new NewPackageRegistration( + "NuGet.Versioning", + 1001, + new string[0], + new[] { new TestPackage("1.0.0") { SemVerLevelKey = SemVerLevelKey.SemVer2 } }, + isExcludedByDefault: shouldBeExcluded); + + var actions = _target.AddNewPackageRegistration(input); + + _search.Verify( + x => x.FullFromDb( + input.PackageId, + SearchFilters.IncludePrereleaseAndSemVer2, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + input.Packages[0], + input.Owners, + input.TotalDownloadCount, + shouldBeExcluded), + Times.Once); + } + + [Fact] + public void ReturnsDeleteSearchActionsForAllUnlisted() + { + var input = new NewPackageRegistration( + "NuGet.Versioning", + 1001, + new string[0], + new[] { new TestPackage("1.0.0") { Listed = false } }, + false); + + var actions = _target.AddNewPackageRegistration(input); + + Assert.Equal(4, actions.Search.Count); + Assert.Equal(IndexActionType.Delete, actions.Search[0].ActionType); // SearchFilters.Default + Assert.Equal(IndexActionType.Delete, actions.Search[1].ActionType); // SearchFilters.IncludePrerelease + Assert.Equal(IndexActionType.Delete, actions.Search[2].ActionType); // SearchFilters.IncludeSemVer2 + Assert.Equal(IndexActionType.Delete, actions.Search[3].ActionType); // SearchFilters.IncludePrereleaseAndSemVer2 + } + + [Fact] + public void UsesSemVerLevelToIndicateSemVer2() + { + _search + .Setup(x => x.FullFromDb( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(() => new SearchDocument.Full()); + var input = new NewPackageRegistration( + "NuGet.Versioning", + 1001, + new string[0], + new[] { new TestPackage("1.0.0") { SemVerLevelKey = SemVerLevelKey.SemVer2 } }, + false); + + var actions = _target.AddNewPackageRegistration(input); + + Assert.Equal(4, actions.Search.Count); + Assert.Equal(IndexActionType.Delete, actions.Search[0].ActionType); // SearchFilters.Default + Assert.Equal(IndexActionType.Delete, actions.Search[1].ActionType); // SearchFilters.IncludePrerelease + Assert.Equal(IndexActionType.Upload, actions.Search[2].ActionType); // SearchFilters.IncludeSemVer2 + Assert.IsType(actions.Search[2].Document); + Assert.Equal(IndexActionType.Upload, actions.Search[3].ActionType); // SearchFilters.IncludePrereleaseAndSemVer2 + Assert.IsType(actions.Search[3].Document); + } + } + + /// + /// Provides a convenience constructor that initialized version-related properties in a consistent manner. + /// + public class TestPackage : Package + { + public TestPackage(string version) + { + var parsedVersion = NuGetVersion.Parse(version); + Version = version; + NormalizedVersion = parsedVersion.ToNormalizedString(); + IsPrerelease = parsedVersion.IsPrerelease; + } + } + + public abstract class BaseFacts + { + protected readonly Mock _search; + protected readonly Mock _hijack; + protected readonly RecordingLogger _logger; + protected readonly PackageEntityIndexActionBuilder _target; + + public BaseFacts(ITestOutputHelper output) + { + _search = new Mock(); + _hijack = new Mock(); + _logger = output.GetLogger(); + + _search + .Setup(x => x.LatestFlagsOrNull(It.IsAny(), It.IsAny())) + .Returns((vl, sf) => new SearchDocument.LatestFlags( + vl.GetLatestVersionInfoOrNull(sf), + isLatestStable: true, + isLatest: true)); + + _target = new PackageEntityIndexActionBuilder( + _search.Object, + _hijack.Object, + _logger); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/DependencyInjectionExtensionsFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/DependencyInjectionExtensionsFacts.cs new file mode 100644 index 000000000..df79e5360 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/DependencyInjectionExtensionsFacts.cs @@ -0,0 +1,161 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Search; +using Microsoft.Extensions.Logging; +using Microsoft.Rest.TransientFaultHandling; +using Moq; +using Moq.Language.Flow; +using NuGet.Services.AzureSearch.Wrappers; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch +{ + public class DependencyInjectionExtensionsFacts + { + public class TheGetRetryPolicyMethod + { + public TheGetRetryPolicyMethod(ITestOutputHelper output) + { + LoggerFactory = new LoggerFactory().AddXunit(output); + HttpClientHandler = new Mock { CallBase = true }; + + SearchServiceClient = new SearchServiceClient( + "test-search-service", + new SearchCredentials("api-key"), + HttpClientHandler.Object, + DependencyInjectionExtensions.GetSearchDelegatingHandlers(LoggerFactory)); + SearchServiceClient.SetRetryPolicy(SingleRetry); + + IndexesOperationsWrapper = new IndexesOperationsWrapper( + SearchServiceClient.Indexes, + DependencyInjectionExtensions.GetSearchDelegatingHandlers(LoggerFactory), + SingleRetry, + LoggerFactory.CreateLogger()); + } + + [Theory] + [MemberData(nameof(NonTransientTestData))] + public async Task DoesNotRetryNonTransientErrorsForIndexOperations(Action>> setup) + { + setup(HttpClientHandler.Setup(x => x.OnSendAsync(It.IsAny(), It.IsAny()))); + + await Assert.ThrowsAnyAsync(() => SearchServiceClient.Indexes.ListAsync()); + + VerifyAttemptCount(1); + } + + [Theory] + [MemberData(nameof(NonTransientTestData))] + public async Task DoesNotRetryNonTransientErrorsForDocumentOperations(Action>> setup) + { + setup(HttpClientHandler.Setup(x => x.OnSendAsync(It.IsAny(), It.IsAny()))); + + await Assert.ThrowsAnyAsync(() => IndexesOperationsWrapper.GetClient("test-index").Documents.CountAsync()); + + VerifyAttemptCount(1); + } + + [Theory] + [MemberData(nameof(TransientTestData))] + public async Task RetriesTransientErrorsForIndexOperations(Action>> setup) + { + setup(HttpClientHandler.Setup(x => x.OnSendAsync(It.IsAny(), It.IsAny()))); + + await Assert.ThrowsAnyAsync(() => SearchServiceClient.Indexes.ListAsync()); + + VerifyAttemptCount(2); + } + + [Theory] + [MemberData(nameof(TransientTestData))] + public async Task RetriesTransientErrorsForDocumentOperations(Action>> setup) + { + setup(HttpClientHandler.Setup(x => x.OnSendAsync(It.IsAny(), It.IsAny()))); + + await Assert.ThrowsAnyAsync(() => IndexesOperationsWrapper.GetClient("test-index").Documents.CountAsync()); + + VerifyAttemptCount(2); + } + + private void VerifyAttemptCount(int count) + { + HttpClientHandler.Verify( + x => x.OnSendAsync(It.IsAny(), It.IsAny()), + Times.Exactly(count)); + } + + public static IEnumerable TransientHttpStatusCodes => new[] + { + HttpStatusCode.RequestTimeout, + (HttpStatusCode)429, + HttpStatusCode.InternalServerError, + HttpStatusCode.BadGateway, + HttpStatusCode.InternalServerError, + HttpStatusCode.GatewayTimeout, + }; + + public static IEnumerable TransientWebExceptionStatuses => new[] + { + WebExceptionStatus.ConnectFailure, + WebExceptionStatus.ConnectionClosed, + WebExceptionStatus.KeepAliveFailure, + WebExceptionStatus.ReceiveFailure, + }; + + public static IEnumerable TransientTestData => GetTestData(TransientHttpStatusCodes, TransientWebExceptionStatuses); + + public static IEnumerable NonTransientHttpStatusCodes => new[] + { + HttpStatusCode.BadRequest, + HttpStatusCode.Unauthorized, + HttpStatusCode.Forbidden, + HttpStatusCode.NotFound, + HttpStatusCode.Conflict, + HttpStatusCode.NotImplemented, + }; + + public static IEnumerable NonTransientWebExceptionStatuses => new[] + { + WebExceptionStatus.TrustFailure, + WebExceptionStatus.NameResolutionFailure, + }; + + public static IEnumerable NonTransientTestData => GetTestData(NonTransientHttpStatusCodes, NonTransientWebExceptionStatuses); + + private static IEnumerable GetTestData(IEnumerable statusCodes, IEnumerable webExceptionStatuses) + { + var setups = new List>>>(); + + foreach (var statusCode in statusCodes) + { + setups.Add(s => s.ReturnsAsync(() => new HttpResponseMessage(statusCode) { Content = new StringContent(string.Empty) })); + } + + foreach (var webExceptionStatus in webExceptionStatuses) + { + setups.Add(s => s.ThrowsAsync(new HttpRequestException("Fail.", new WebException("Inner fail.", webExceptionStatus)))); + } + + return setups.Select(x => new object[] { x }); + } + + public RetryPolicy SingleRetry => new RetryPolicy( + new HttpStatusCodeErrorDetectionStrategy(), + new FixedIntervalRetryStrategy(retryCount: 1, retryInterval: TimeSpan.Zero)); + + public ILoggerFactory LoggerFactory { get; } + public Mock HttpClientHandler { get; } + public SearchServiceClient SearchServiceClient { get; } + public IndexesOperationsWrapper IndexesOperationsWrapper { get; } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/DocumentUtilitiesFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/DocumentUtilitiesFacts.cs new file mode 100644 index 000000000..691eb6e73 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/DocumentUtilitiesFacts.cs @@ -0,0 +1,63 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using Xunit; + +namespace NuGet.Services.AzureSearch +{ + public class DocumentUtilitiesFacts + { + public class GetHijackDocumentKey + { + [Theory] + [InlineData("NuGet.Versioning", "1.0.0", "nuget_versioning_1_0_0-bnVnZXQudmVyc2lvbmluZy8xLjAuMA2")] + [InlineData("nuget.versioning", "1.0.0", "nuget_versioning_1_0_0-bnVnZXQudmVyc2lvbmluZy8xLjAuMA2")] + [InlineData("NUGET.VERSIONING", "1.0.0", "nuget_versioning_1_0_0-bnVnZXQudmVyc2lvbmluZy8xLjAuMA2")] + [InlineData("_", "1.0.0", "1_0_0-Xy8xLjAuMA2")] + [InlineData("foo-bar", "1.0.0", "foo-bar_1_0_0-Zm9vLWJhci8xLjAuMA2")] + [InlineData("İzmir", "1.0.0", "zmir_1_0_0-xLB6bWlyLzEuMC4w0")] + [InlineData("İİzmir", "1.0.0", "zmir_1_0_0-xLDEsHptaXIvMS4wLjA1")] + [InlineData("zİİmir", "1.0.0", "z__mir_1_0_0-esSwxLBtaXIvMS4wLjA1")] + [InlineData("zmirİ", "1.0.0", "zmir__1_0_0-em1pcsSwLzEuMC4w0")] + [InlineData("zmirİİ", "1.0.0", "zmir___1_0_0-em1pcsSwxLAvMS4wLjA1")] + [InlineData("惡", "1.0.0", "1_0_0-5oOhLzEuMC4w0")] + [InlineData("jQuery", "1.0.0-alpha", "jquery_1_0_0-alpha-anF1ZXJ5LzEuMC4wLWFscGhh0")] + [InlineData("jQuery", "1.0.0-Alpha", "jquery_1_0_0-alpha-anF1ZXJ5LzEuMC4wLWFscGhh0")] + [InlineData("jQuery", "1.0.0-ALPHA", "jquery_1_0_0-alpha-anF1ZXJ5LzEuMC4wLWFscGhh0")] + [InlineData("jQuery", "1.0.0-ALPHA.1", "jquery_1_0_0-alpha_1-anF1ZXJ5LzEuMC4wLWFscGhhLjE1")] + public void EncodesHijackDocumentKey(string id, string version, string expected) + { + var actual = DocumentUtilities.GetHijackDocumentKey(id, version); + + Assert.Equal(expected, actual); + } + } + + public class GetSearchDocumentKey + { + [Theory] + [InlineData("NuGet.Versioning", "nuget_versioning-bnVnZXQudmVyc2lvbmluZw2")] + [InlineData("nuget.versioning", "nuget_versioning-bnVnZXQudmVyc2lvbmluZw2")] + [InlineData("NUGET.VERSIONING", "nuget_versioning-bnVnZXQudmVyc2lvbmluZw2")] + [InlineData("_", "Xw2")] + [InlineData("foo-bar", "foo-bar-Zm9vLWJhcg2")] + [InlineData("İzmir", "zmir-xLB6bWly0")] + [InlineData("İİzmir", "zmir-xLDEsHptaXI1")] + [InlineData("zİİmir", "z__mir-esSwxLBtaXI1")] + [InlineData("zmirİ", "zmir_-em1pcsSw0")] + [InlineData("zmirİİ", "zmir__-em1pcsSwxLA1")] + [InlineData("惡", "5oOh0")] + public void EncodesSearchDocumentKey(string id, string expected) + { + foreach (var searchFilters in Enum.GetValues(typeof(SearchFilters)).Cast()) + { + var actual = DocumentUtilities.GetSearchDocumentKey(id, searchFilters); + + Assert.Equal(expected + "-" + searchFilters, actual); + } + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/DownloadTransferrerFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/DownloadTransferrerFacts.cs new file mode 100644 index 000000000..092d5fde9 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/DownloadTransferrerFacts.cs @@ -0,0 +1,677 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Services.AzureSearch.Auxiliary2AzureSearch; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using Xunit; + +namespace NuGet.Services.AzureSearch +{ + public class DownloadTransferrerFacts + { + public class InitializeDownloadTransfers : Facts + { + [Fact] + public void ReturnsEmptyResult() + { + var result = Target.InitializeDownloadTransfers( + DownloadData, + PopularityTransfers); + + Assert.Empty(result); + } + + [Fact] + public void DoesNothingIfNoTransfers() + { + PopularityTransfer = 0.5; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 5); + + var result = Target.InitializeDownloadTransfers( + DownloadData, + PopularityTransfers); + + Assert.Empty(result); + } + + [Fact] + public void TransfersPopularity() + { + PopularityTransfer = 0.5; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 5); + + PopularityTransfers.AddTransfer("A", "B"); + + var result = Target.InitializeDownloadTransfers( + DownloadData, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B" }, result.Keys); + Assert.Equal(50, result["A"]); + Assert.Equal(55, result["B"]); + } + + [Fact] + public void SplitsPopularity() + { + PopularityTransfer = 0.5; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 5); + DownloadData.SetDownloadCount("C", "1.0.0", 1); + + PopularityTransfers.AddTransfer("A", "B"); + PopularityTransfers.AddTransfer("A", "C"); + + var result = Target.InitializeDownloadTransfers( + DownloadData, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B", "C" }, result.Keys); + Assert.Equal(50, result["A"]); + Assert.Equal(30, result["B"]); + Assert.Equal(26, result["C"]); + } + + [Fact] + public void PopularityTransferRoundsDown() + { + PopularityTransfer = 0.5; + + DownloadData.SetDownloadCount("A", "1.0.0", 3); + DownloadData.SetDownloadCount("B", "1.0.0", 0); + + PopularityTransfers.AddTransfer("A", "B"); + + var result = Target.InitializeDownloadTransfers( + DownloadData, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B" }, result.Keys); + Assert.Equal(1, result["A"]); + Assert.Equal(1, result["B"]); + } + + [Fact] + public void AcceptsPopularityFromMultipleSources() + { + PopularityTransfer = 1; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 20); + DownloadData.SetDownloadCount("C", "1.0.0", 1); + + PopularityTransfers.AddTransfer("A", "C"); + PopularityTransfers.AddTransfer("B", "C"); + + var result = Target.InitializeDownloadTransfers( + DownloadData, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B", "C" }, result.Keys); + Assert.Equal(0, result["A"]); + Assert.Equal(0, result["B"]); + Assert.Equal(121, result["C"]); + } + + [Fact] + public void SupportsZeroPopularityTransfer() + { + PopularityTransfer = 0; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 5); + + PopularityTransfers.AddTransfer("A", "B"); + + var result = Target.InitializeDownloadTransfers( + DownloadData, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B" }, result.Keys); + Assert.Equal(100, result["A"]); + Assert.Equal(5, result["B"]); + } + + [Fact] + public void PackageWithOutgoingTransferRejectsIncomingTransfer() + { + PopularityTransfer = 1; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 0); + DownloadData.SetDownloadCount("C", "1.0.0", 0); + + PopularityTransfers.AddTransfer("A", "B"); + PopularityTransfers.AddTransfer("A", "C"); + PopularityTransfers.AddTransfer("B", "C"); + + var result = Target.InitializeDownloadTransfers( + DownloadData, + PopularityTransfers); + + // B has incoming and outgoing popularity transfers. It should reject the incoming transfer. + Assert.Equal(new[] { "A", "B", "C" }, result.Keys); + Assert.Equal(0, result["A"]); + Assert.Equal(0, result["B"]); + Assert.Equal(50, result["C"]); + } + + [Fact] + public void PopularityTransfersAreNotTransitive() + { + PopularityTransfer = 1; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 100); + DownloadData.SetDownloadCount("C", "1.0.0", 100); + + PopularityTransfers.AddTransfer("A", "B"); + PopularityTransfers.AddTransfer("B", "C"); + + var result = Target.InitializeDownloadTransfers( + DownloadData, + PopularityTransfers); + + // A transfers downloads to B. + // B transfers downloads to C. + // B and C should reject downloads from A. + Assert.Equal(new[] { "A", "B", "C" }, result.Keys); + Assert.Equal(0, result["A"]); + Assert.Equal(0, result["B"]); + Assert.Equal(200, result["C"]); + } + + [Fact] + public void RejectsCyclicalPopularityTransfers() + { + PopularityTransfer = 1; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 100); + + PopularityTransfers.AddTransfer("A", "B"); + PopularityTransfers.AddTransfer("B", "A"); + + var result = Target.InitializeDownloadTransfers( + DownloadData, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B" }, result.Keys); + Assert.Equal(0, result["A"]); + Assert.Equal(0, result["B"]); + } + + [Fact] + public void UnknownPackagesTransferZeroDownloads() + { + PopularityTransfer = 1; + + PopularityTransfers.AddTransfer("A", "B"); + + var result = Target.InitializeDownloadTransfers( + DownloadData, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B" }, result.Keys); + Assert.Equal(0, result["A"]); + Assert.Equal(0, result["B"]); + } + } + + public class GetUpdatedTransferChanges : Facts + { + [Fact] + public void RequiresDownloadDataForDownloadChange() + { + DownloadChanges["A"] = 1; + + var ex = Assert.Throws( + () => Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers)); + + Assert.Equal("The download changes should match the latest downloads", ex.Message); + } + + [Fact] + public void RequiresDownloadDataAndChangesMatch() + { + DownloadData.SetDownloadCount("A", "1.0.0", 1); + DownloadChanges["A"] = 2; + + var ex = Assert.Throws( + () => Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers)); + + Assert.Equal("The download changes should match the latest downloads", ex.Message); + } + + [Fact] + public void ReturnsEmptyResult() + { + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + Assert.Empty(result); + } + + [Fact] + public void DoesNothingIfNoTransfers() + { + PopularityTransfer = 0.5; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 5); + + DownloadChanges["A"] = 100; + DownloadChanges["B"] = 5; + + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + Assert.Empty(result); + } + + [Fact] + public void DoesNothingIfNoChanges() + { + PopularityTransfer = 0.5; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 5); + + PopularityTransfers.AddTransfer("A", "B"); + + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + Assert.Empty(result); + } + + [Fact] + public void OutgoingTransfersNewDownloads() + { + PopularityTransfer = 1; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 20); + DownloadData.SetDownloadCount("C", "1.0.0", 1); + + DownloadChanges["A"] = 100; + + PopularityTransfers.AddTransfer("A", "C"); + PopularityTransfers.AddTransfer("B", "C"); + + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + // C receives downloads from A and B + // A has download changes + // B has no changes + Assert.Equal(new[] { "A", "C" }, result.Keys); + Assert.Equal(0, result["A"]); + Assert.Equal(121, result["C"]); + } + + [Fact] + public void OutgoingTransfersSplitsNewDownloads() + { + PopularityTransfer = 1; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 5); + DownloadData.SetDownloadCount("C", "1.0.0", 0); + + DownloadChanges["A"] = 100; + + PopularityTransfers.AddTransfer("A", "B"); + PopularityTransfers.AddTransfer("A", "C"); + + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B", "C" }, result.Keys); + Assert.Equal(0, result["A"]); + Assert.Equal(55, result["B"]); + Assert.Equal(50, result["C"]); + } + + [Fact] + public void PopularityTransferRoundsDown() + { + PopularityTransfer = 0.5; + + DownloadData.SetDownloadCount("A", "1.0.0", 3); + DownloadData.SetDownloadCount("B", "1.0.0", 0); + + DownloadChanges["A"] = 3; + + PopularityTransfers.AddTransfer("A", "B"); + + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B" }, result.Keys); + Assert.Equal(1, result["A"]); + Assert.Equal(1, result["B"]); + } + + [Fact] + public void IncomingTransfersAddedToNewDownloads() + { + PopularityTransfer = 1; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 5); + DownloadData.SetDownloadCount("C", "1.0.0", 0); + + DownloadChanges["B"] = 5; + + PopularityTransfers.AddTransfer("A", "B"); + PopularityTransfers.AddTransfer("A", "C"); + + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + // B has new downloads and receives downloads from A. + Assert.Equal(new[] { "B" }, result.Keys); + Assert.Equal(55, result["B"]); + } + + [Fact] + public void NewOrUpdatedPopularityTransfer() + { + PopularityTransfer = 1; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 5); + + PopularityTransfers.AddTransfer("A", "B"); + + TransferChanges["A"] = new[] { "B" }; + + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B" }, result.Keys); + Assert.Equal(0, result["A"]); + Assert.Equal(105, result["B"]); + } + + [Fact] + public void NewOrUpdatedSplitsPopularityTransfer() + { + PopularityTransfer = 1; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 5); + DownloadData.SetDownloadCount("C", "1.0.0", 0); + + PopularityTransfers.AddTransfer("A", "B"); + PopularityTransfers.AddTransfer("A", "C"); + + TransferChanges["A"] = new[] { "B", "C" }; + + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B", "C" }, result.Keys); + Assert.Equal(0, result["A"]); + Assert.Equal(55, result["B"]); + Assert.Equal(50, result["C"]); + } + + [Fact] + public void RemovesIncomingPopularityTransfer() + { + // A used to transfer to both B and C. + // A now transfers to just B. + PopularityTransfer = 1; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 5); + DownloadData.SetDownloadCount("C", "1.0.0", 0); + + PopularityTransfers.AddTransfer("A", "B"); + + TransferChanges["A"] = new[] { "B" }; + OldTransfers.AddTransfer("A", "B"); + OldTransfers.AddTransfer("A", "C"); + + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B", "C" }, result.Keys); + Assert.Equal(0, result["A"]); + Assert.Equal(105, result["B"]); + Assert.Equal(0, result["C"]); + } + + [Fact] + public void RemovePopularityTransfer() + { + // A used to transfer to both B and C. + PopularityTransfer = 1; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 5); + DownloadData.SetDownloadCount("C", "1.0.0", 0); + + TransferChanges["A"] = new string[0]; + OldTransfers.AddTransfer("A", "B"); + OldTransfers.AddTransfer("A", "C"); + + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B", "C" }, result.Keys); + Assert.Equal(100, result["A"]); + Assert.Equal(5, result["B"]); + Assert.Equal(0, result["C"]); + } + + [Fact] + public void SupportsZeroPopularityTransfer() + { + PopularityTransfer = 0; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 5); + + DownloadChanges["A"] = 100; + + PopularityTransfers.AddTransfer("A", "B"); + + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B" }, result.Keys); + Assert.Equal(100, result["A"]); + Assert.Equal(5, result["B"]); + } + + [Fact] + public void PackageWithOutgoingTransferRejectsIncomingTransfer() + { + PopularityTransfer = 1; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 0); + DownloadData.SetDownloadCount("C", "1.0.0", 0); + + DownloadChanges["A"] = 100; + + PopularityTransfers.AddTransfer("A", "B"); + PopularityTransfers.AddTransfer("A", "C"); + PopularityTransfers.AddTransfer("B", "C"); + + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + // B has incoming and outgoing popularity transfers. It should reject the incoming transfer. + Assert.Equal(new[] { "A", "B", "C" }, result.Keys); + Assert.Equal(0, result["A"]); + Assert.Equal(0, result["B"]); + Assert.Equal(50, result["C"]); + } + + [Fact] + public void PopularityTransfersAreNotTransitive() + { + PopularityTransfer = 1; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 100); + DownloadData.SetDownloadCount("C", "1.0.0", 100); + + DownloadChanges["A"] = 100; + + PopularityTransfers.AddTransfer("A", "B"); + PopularityTransfers.AddTransfer("B", "C"); + + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + // A transfers downloads to B. + // B transfers downloads to C. + // B and C should reject downloads from A. + Assert.Equal(new[] { "A", "B" }, result.Keys); + Assert.Equal(0, result["A"]); + Assert.Equal(0, result["B"]); + } + + [Fact] + public void RejectsCyclicalPopularityTransfers() + { + PopularityTransfer = 1; + + DownloadData.SetDownloadCount("A", "1.0.0", 100); + DownloadData.SetDownloadCount("B", "1.0.0", 100); + + DownloadChanges["A"] = 100; + DownloadChanges["B"] = 100; + + PopularityTransfers.AddTransfer("A", "B"); + PopularityTransfers.AddTransfer("B", "A"); + + var result = Target.UpdateDownloadTransfers( + DownloadData, + DownloadChanges, + OldTransfers, + PopularityTransfers); + + Assert.Equal(new[] { "A", "B" }, result.Keys); + Assert.Equal(0, result["A"]); + Assert.Equal(0, result["B"]); + } + + public GetUpdatedTransferChanges() + { + DownloadChanges = new SortedDictionary(StringComparer.OrdinalIgnoreCase); + OldTransfers = new PopularityTransferData(); + } + + public SortedDictionary DownloadChanges { get; } + public PopularityTransferData OldTransfers { get; } + } + + public class Facts + { + public Facts() + { + TransferChanges = new SortedDictionary(); + DataComparer = new Mock(); + DataComparer + .Setup(x => x.ComparePopularityTransfers( + It.IsAny(), + It.IsAny())) + .Returns(TransferChanges); + + PopularityTransfers = new PopularityTransferData(); + + var options = new Mock>(); + options + .Setup(x => x.Value) + .Returns(() => new AzureSearchJobConfiguration + { + Scoring = new AzureSearchScoringConfiguration + { + PopularityTransfer = PopularityTransfer + } + }); + + DownloadData = new DownloadData(); + + Target = new DownloadTransferrer( + DataComparer.Object, + options.Object, + Mock.Of>()); + } + + public Mock DataComparer { get; } + public IDownloadTransferrer Target { get; } + + public DownloadData DownloadData { get; } + public PopularityTransferData PopularityTransfers { get; } + public SortedDictionary TransferChanges { get; } + public double PopularityTransfer = 0; + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/HijackDocumentBuilderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/HijackDocumentBuilderFacts.cs new file mode 100644 index 000000000..e420f91e2 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/HijackDocumentBuilderFacts.cs @@ -0,0 +1,487 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Services.AzureSearch.Support; +using NuGet.Services.Entities; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch +{ + public class HijackDocumentBuilderFacts + { + public class Keyed : BaseFacts + { + public Keyed(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task SetsExpectedProperties() + { + var document = _target.Keyed(Data.PackageId, Data.NormalizedVersion); + + var json = await SerializationUtilities.SerializeToJsonAsync(document); + Assert.Equal(@"{ + ""value"": [ + { + ""@search.action"": ""upload"", + ""key"": ""windowsazure_storage_7_1_2-alpha-d2luZG93c2F6dXJlLnN0b3JhZ2UvNy4xLjItYWxwaGE1"" + } + ] +}", json); + } + } + + public class LatestFromCatalog : BaseFacts + { + public LatestFromCatalog(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task SetsExpectedProperties() + { + var document = _target.LatestFromCatalog( + Data.PackageId, + Data.NormalizedVersion, + Data.CommitTimestamp, + Data.CommitId, + Data.HijackDocumentChanges); + + SetDocumentLastUpdated(document); + var json = await SerializationUtilities.SerializeToJsonAsync(document); + Assert.Equal(@"{ + ""value"": [ + { + ""@search.action"": ""upload"", + ""isLatestStableSemVer1"": false, + ""isLatestSemVer1"": true, + ""isLatestStableSemVer2"": false, + ""isLatestSemVer2"": true, + ""lastCommitTimestamp"": ""2018-12-13T12:30:00+00:00"", + ""lastCommitId"": ""6b9b24dd-7aec-48ae-afc1-2a117e3d50d1"", + ""lastUpdatedDocument"": ""2018-12-14T09:30:00+00:00"", + ""lastDocumentType"": ""NuGet.Services.AzureSearch.HijackDocument+Latest"", + ""lastUpdatedFromCatalog"": true, + ""key"": ""windowsazure_storage_7_1_2-alpha-d2luZG93c2F6dXJlLnN0b3JhZ2UvNy4xLjItYWxwaGE1"" + } + ] +}", json); + } + } + + public class FullFromDb : BaseFacts + { + public FullFromDb(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public void LeavesNullTagsAsNull() + { + var package = Data.PackageEntity; + package.Tags = null; + + var document = _target.FullFromDb(Data.PackageId, Data.HijackDocumentChanges, package); + + Assert.Null(document.Tags); + } + + [Fact] + public async Task SerializesNullSemVerLevel() + { + var package = Data.PackageEntity; + package.SemVerLevelKey = SemVerLevelKey.Unknown; + + var document = _target.FullFromDb(Data.PackageId, Data.HijackDocumentChanges, package); + + var json = await SerializationUtilities.SerializeToJsonAsync(document); + Assert.Contains("\"semVerLevel\": null,", json); + } + + [Fact] + public async Task SetsExpectedProperties() + { + var document = _target.FullFromDb(Data.PackageId, Data.HijackDocumentChanges, Data.PackageEntity); + + SetDocumentLastUpdated(document); + var json = await SerializationUtilities.SerializeToJsonAsync(document); + Assert.Equal(@"{ + ""value"": [ + { + ""@search.action"": ""upload"", + ""listed"": true, + ""isLatestStableSemVer1"": false, + ""isLatestSemVer1"": true, + ""isLatestStableSemVer2"": false, + ""isLatestSemVer2"": true, + ""semVerLevel"": 2, + ""authors"": ""Microsoft"", + ""copyright"": ""© Microsoft Corporation. All rights reserved."", + ""created"": ""2017-01-01T00:00:00+00:00"", + ""description"": ""Description."", + ""fileSize"": 3039254, + ""flattenedDependencies"": ""Microsoft.Data.OData:5.6.4:net40-client|Newtonsoft.Json:6.0.8:net40-client"", + ""hash"": ""oMs9XKzRTsbnIpITcqZ5XAv1h2z6oyJ33+Z/PJx36iVikge/8wm5AORqAv7soKND3v5/0QWW9PQ0ktQuQu9aQQ=="", + ""hashAlgorithm"": ""SHA512"", + ""iconUrl"": ""http://go.microsoft.com/fwlink/?LinkID=288890"", + ""language"": ""en-US"", + ""lastEdited"": ""2017-01-02T00:00:00+00:00"", + ""licenseUrl"": ""http://go.microsoft.com/fwlink/?LinkId=331471"", + ""minClientVersion"": ""2.12"", + ""normalizedVersion"": ""7.1.2-alpha"", + ""originalVersion"": ""7.1.2.0-alpha+git"", + ""packageId"": ""WindowsAzure.Storage"", + ""prerelease"": true, + ""projectUrl"": ""https://github.com/Azure/azure-storage-net"", + ""published"": ""2017-01-03T00:00:00+00:00"", + ""releaseNotes"": ""Release notes."", + ""requiresLicenseAcceptance"": true, + ""sortableTitle"": ""windows azure storage"", + ""summary"": ""Summary."", + ""tags"": [ + ""Microsoft"", + ""Azure"", + ""Storage"", + ""Table"", + ""Blob"", + ""File"", + ""Queue"", + ""Scalable"", + ""windowsazureofficial"" + ], + ""title"": ""Windows Azure Storage"", + ""tokenizedPackageId"": ""WindowsAzure.Storage"", + ""lastCommitTimestamp"": null, + ""lastCommitId"": null, + ""lastUpdatedDocument"": ""2018-12-14T09:30:00+00:00"", + ""lastDocumentType"": ""NuGet.Services.AzureSearch.HijackDocument+Full"", + ""lastUpdatedFromCatalog"": false, + ""key"": ""windowsazure_storage_7_1_2-alpha-d2luZG93c2F6dXJlLnN0b3JhZ2UvNy4xLjItYWxwaGE1"" + } + ] +}", json); + } + + [Theory] + [MemberData(nameof(MissingTitles))] + public void UsesIdWhenMissingForTitle(string title) + { + var package = Data.PackageEntity; + package.Title = title; + + var document = _target.FullFromDb(Data.PackageId, Data.HijackDocumentChanges, package); + + Assert.Equal(Data.PackageId, document.Title); + } + + [Theory] + [MemberData(nameof(MissingTitles))] + public void UsesLowerIdWhenMissingForSortableTitle(string title) + { + var package = Data.PackageEntity; + package.Title = title; + + var document = _target.FullFromDb(Data.PackageId, Data.HijackDocumentChanges, package); + + Assert.Equal(Data.PackageId.ToLowerInvariant(), document.SortableTitle); + } + + [Fact] + public void Uses1900ForPublishedWhenUnlisted() + { + var package = Data.PackageEntity; + package.Listed = false; + + var document = _target.FullFromDb(Data.PackageId, Data.HijackDocumentChanges, package); + + Assert.Equal(DateTimeOffset.Parse("1900-01-01Z"), document.Published); + } + + [Fact] + public void SplitsTags() + { + var package = Data.PackageEntity; + package.Tags = "foo; BAR | Baz"; + + var document = _target.FullFromDb(Data.PackageId, Data.HijackDocumentChanges, package); + + Assert.Equal(new[] { "foo", "BAR", "Baz" }, document.Tags); + } + + [Theory] + [InlineData(null)] + [InlineData(SemVerLevelKey.SemVer2)] + public void UsesSemVerLevelToIndicateSemVer2(int? semVerLevelKey) + { + var package = Data.PackageEntity; + package.SemVerLevelKey = semVerLevelKey; + + var document = _target.FullFromDb(Data.PackageId, Data.HijackDocumentChanges, package); + + Assert.Equal(semVerLevelKey, document.SemVerLevel); + } + + /// + /// The caller is expected to verify consistency. + /// + [Fact] + public void DoesNotUseVersionToIndicateIsPrerelease() + { + var package = Data.PackageEntity; + package.IsPrerelease = false; + package.Version = "2.0.0-alpha"; + package.NormalizedVersion = "2.0.0-alpha"; + + var document = _target.FullFromDb(Data.PackageId, Data.HijackDocumentChanges, package); + + Assert.False(document.Prerelease); + } + + [Fact] + public void SetsLicenseUrlToGalleryWhenPackageHasLicenseExpression() + { + var package = Data.PackageEntity; + package.LicenseExpression = "MIT"; + + var document = _target.FullFromDb(Data.PackageId, Data.HijackDocumentChanges, package); + + Assert.Equal(Data.GalleryLicenseUrl, document.LicenseUrl); + } + + [Theory] + [InlineData(EmbeddedLicenseFileType.PlainText)] + [InlineData(EmbeddedLicenseFileType.Markdown)] + public void SetsLicenseUrlToGalleryWhenPackageHasLicenseFile(EmbeddedLicenseFileType type) + { + var package = Data.PackageEntity; + package.EmbeddedLicenseType = type; + + var document = _target.FullFromDb(Data.PackageId, Data.HijackDocumentChanges, package); + + Assert.Equal(Data.GalleryLicenseUrl, document.LicenseUrl); + } + } + + public class FullFromCatalog : BaseFacts + { + public FullFromCatalog(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task SetsExpectedProperties() + { + var document = _target.FullFromCatalog(Data.NormalizedVersion, Data.HijackDocumentChanges, Data.Leaf); + + SetDocumentLastUpdated(document); + var json = await SerializationUtilities.SerializeToJsonAsync(document); + Assert.Equal(@"{ + ""value"": [ + { + ""@search.action"": ""upload"", + ""listed"": true, + ""isLatestStableSemVer1"": false, + ""isLatestSemVer1"": true, + ""isLatestStableSemVer2"": false, + ""isLatestSemVer2"": true, + ""semVerLevel"": 2, + ""authors"": ""Microsoft"", + ""copyright"": ""© Microsoft Corporation. All rights reserved."", + ""created"": ""2017-01-01T00:00:00+00:00"", + ""description"": ""Description."", + ""fileSize"": 3039254, + ""flattenedDependencies"": ""Microsoft.Data.OData:5.6.4:net40-client|Newtonsoft.Json:6.0.8:net40-client"", + ""hash"": ""oMs9XKzRTsbnIpITcqZ5XAv1h2z6oyJ33+Z/PJx36iVikge/8wm5AORqAv7soKND3v5/0QWW9PQ0ktQuQu9aQQ=="", + ""hashAlgorithm"": ""SHA512"", + ""iconUrl"": ""http://go.microsoft.com/fwlink/?LinkID=288890"", + ""language"": ""en-US"", + ""lastEdited"": ""2017-01-02T00:00:00+00:00"", + ""licenseUrl"": ""http://go.microsoft.com/fwlink/?LinkId=331471"", + ""minClientVersion"": ""2.12"", + ""normalizedVersion"": ""7.1.2-alpha"", + ""originalVersion"": ""7.1.2.0-alpha+git"", + ""packageId"": ""WindowsAzure.Storage"", + ""prerelease"": true, + ""projectUrl"": ""https://github.com/Azure/azure-storage-net"", + ""published"": ""2017-01-03T00:00:00+00:00"", + ""releaseNotes"": ""Release notes."", + ""requiresLicenseAcceptance"": true, + ""sortableTitle"": ""windows azure storage"", + ""summary"": ""Summary."", + ""tags"": [ + ""Microsoft"", + ""Azure"", + ""Storage"", + ""Table"", + ""Blob"", + ""File"", + ""Queue"", + ""Scalable"", + ""windowsazureofficial"" + ], + ""title"": ""Windows Azure Storage"", + ""tokenizedPackageId"": ""WindowsAzure.Storage"", + ""lastCommitTimestamp"": ""2018-12-13T12:30:00+00:00"", + ""lastCommitId"": ""6b9b24dd-7aec-48ae-afc1-2a117e3d50d1"", + ""lastUpdatedDocument"": ""2018-12-14T09:30:00+00:00"", + ""lastDocumentType"": ""NuGet.Services.AzureSearch.HijackDocument+Full"", + ""lastUpdatedFromCatalog"": true, + ""key"": ""windowsazure_storage_7_1_2-alpha-d2luZG93c2F6dXJlLnN0b3JhZ2UvNy4xLjItYWxwaGE1"" + } + ] +}", json); + } + + [Fact] + public void ConsidersPublished1900AsUnlisted() + { + var leaf = Data.Leaf; + leaf.Listed = null; + leaf.Published = DateTimeOffset.Parse("1900-01-01Z"); + + var document = _target.FullFromCatalog(Data.NormalizedVersion, Data.HijackDocumentChanges, leaf); + + Assert.False(document.Listed); + } + + [Theory] + [MemberData(nameof(MissingTitles))] + public void UsesIdWhenMissingForTitle(string title) + { + var leaf = Data.Leaf; + leaf.Title = title; + + var document = _target.FullFromCatalog(Data.NormalizedVersion, Data.HijackDocumentChanges, leaf); + + Assert.Equal(Data.PackageId, document.Title); + } + + [Theory] + [MemberData(nameof(MissingTitles))] + public void UsesLowerIdWhenMissingForSortableTitle(string title) + { + var leaf = Data.Leaf; + leaf.Title = title; + + var document = _target.FullFromCatalog(Data.NormalizedVersion, Data.HijackDocumentChanges, leaf); + + Assert.Equal(Data.PackageId.ToLowerInvariant(), document.SortableTitle); + } + + [Fact] + public void LeavesNullRequiresLicenseAcceptanceAsNull() + { + var leaf = Data.Leaf; + leaf.RequireLicenseAcceptance = null; + + var document = _target.FullFromCatalog(Data.NormalizedVersion, Data.HijackDocumentChanges, leaf); + + Assert.Null(document.RequiresLicenseAcceptance); + } + + [Fact] + public void LeavesNullVerbatimVersionAsNull() + { + var leaf = Data.Leaf; + leaf.VerbatimVersion = null; + + var document = _target.FullFromCatalog(Data.NormalizedVersion, Data.HijackDocumentChanges, leaf); + + Assert.Null(document.OriginalVersion); + } + + [Fact] + public void SetsLicenseUrlToGalleryWhenPackageHasLicenseExpression() + { + var leaf = Data.Leaf; + leaf.LicenseExpression = "MIT"; + + var document = _target.FullFromCatalog(Data.NormalizedVersion, Data.HijackDocumentChanges, leaf); + + Assert.Equal(Data.GalleryLicenseUrl, document.LicenseUrl); + } + + [Fact] + public void SetsLicenseUrlToGalleryWhenPackageHasLicenseFile() + { + var leaf = Data.Leaf; + leaf.LicenseFile = "LICENSE.txt"; + + var document = _target.FullFromCatalog(Data.NormalizedVersion, Data.HijackDocumentChanges, leaf); + + Assert.Equal(Data.GalleryLicenseUrl, document.LicenseUrl); + } + + [Fact] + public void SetsIconUrlToFlatContainerWhenPackageHasIconFileAndIconUrl() + { + var leaf = Data.Leaf; + leaf.IconUrl = "https://other-example/icon.png"; + leaf.IconFile = "icon.png"; + + var document = _target.FullFromCatalog(Data.NormalizedVersion, Data.HijackDocumentChanges, leaf); + + Assert.Equal(Data.FlatContainerIconUrl, document.IconUrl); + } + + [Fact] + public void SetsIconUrlToFlatContainerWhenPackageHasIconFileAndNoIconUrl() + { + var leaf = Data.Leaf; + leaf.IconUrl = null; + leaf.IconFile = "icon.png"; + + var document = _target.FullFromCatalog(Data.NormalizedVersion, Data.HijackDocumentChanges, leaf); + + Assert.Equal(Data.FlatContainerIconUrl, document.IconUrl); + } + } + + public abstract class BaseFacts + { + protected readonly ITestOutputHelper _output; + protected readonly Mock> _options; + protected readonly BaseDocumentBuilder _baseDocumentBuilder; + protected readonly AzureSearchJobConfiguration _config; + protected readonly HijackDocumentBuilder _target; + + public static IEnumerable MissingTitles = new[] + { + new object[] { null }, + new object[] { string.Empty }, + new object[] { " " }, + new object[] { " \t"}, + }; + + public void SetDocumentLastUpdated(IUpdatedDocument document) + { + Data.SetDocumentLastUpdated(document, _output); + } + + public BaseFacts(ITestOutputHelper output) + { + _output = output; + _options = new Mock>(); + _baseDocumentBuilder = new BaseDocumentBuilder(_options.Object); // We intentionally don't mock this. + _config = new AzureSearchJobConfiguration + { + GalleryBaseUrl = Data.GalleryBaseUrl, + FlatContainerBaseUrl = Data.FlatContainerBaseUrl, + FlatContainerContainerName = Data.FlatContainerContainerName, + }; + + _options.Setup(o => o.Value).Returns(() => _config); + + _target = new HijackDocumentBuilder(_baseDocumentBuilder); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/IndexBuilderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/IndexBuilderFacts.cs new file mode 100644 index 000000000..9c91396ca --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/IndexBuilderFacts.cs @@ -0,0 +1,497 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Moq; +using NuGet.Services.AzureSearch.ScoringProfiles; +using NuGet.Services.AzureSearch.SearchService; +using NuGet.Services.AzureSearch.Wrappers; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch +{ + public class BlobContainerBuilderFacts + { + public class CreateAsync : BaseFacts + { + public CreateAsync(ITestOutputHelper output) : base(output) + { + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task CreatesIndex(bool retryOnConflict) + { + await _target.CreateAsync(retryOnConflict); + + VerifyGetContainer(); + _cloudBlobContainer.Verify(x => x.CreateAsync(), Times.Once); + _cloudBlobContainer.Verify(x => x.CreateIfNotExistAsync(), Times.Never); + _cloudBlobContainer.Verify(x => x.DeleteIfExistsAsync(), Times.Never); + VerifySetPermissions(); + } + + [Fact] + public async Task CanRetryOnConflict() + { + EnableConflict(); + + var sw = Stopwatch.StartNew(); + await _target.CreateAsync(retryOnConflict: true); + sw.Stop(); + + _cloudBlobContainer.Verify(x => x.CreateAsync(), Times.Exactly(2)); + VerifySetPermissions(); + Assert.InRange(sw.Elapsed, _retryDuration, TimeSpan.MaxValue); + } + + [Fact] + public async Task CanFailOnConflict() + { + EnableConflict(); + + await Assert.ThrowsAsync(() => _target.CreateAsync(retryOnConflict: false)); + } + } + + public class CreateIfNotExistsAsync : BaseFacts + { + public CreateIfNotExistsAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task CreatesIndexIfNotExists() + { + _cloudBlobContainer.Setup(x => x.ExistsAsync(null, null)).ReturnsAsync(false); + + await _target.CreateIfNotExistsAsync(); + + VerifyGetContainer(); + _cloudBlobContainer.Verify(x => x.CreateAsync(), Times.Once); + _cloudBlobContainer.Verify(x => x.CreateIfNotExistAsync(), Times.Never); + _cloudBlobContainer.Verify(x => x.DeleteIfExistsAsync(), Times.Never); + VerifySetPermissions(); + } + + [Fact] + public async Task DoesNotCreateIndexIfExists() + { + _cloudBlobContainer.Setup(x => x.ExistsAsync(null, null)).ReturnsAsync(true); + + await _target.CreateIfNotExistsAsync(); + + _cloudBlobContainer.Verify(x => x.CreateAsync(), Times.Never); + _cloudBlobContainer.Verify(x => x.CreateIfNotExistAsync(), Times.Never); + _cloudBlobContainer.Verify(x => x.SetPermissionsAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task DoesNotRetryOnConflict() + { + EnableConflict(); + _cloudBlobContainer.Setup(x => x.ExistsAsync(null, null)).ReturnsAsync(false); + + await Assert.ThrowsAsync(() => _target.CreateIfNotExistsAsync()); + } + } + + public class DeleteIfExistsAsync : BaseFacts + { + public DeleteIfExistsAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task Deletes() + { + await _target.DeleteIfExistsAsync(); + + VerifyGetContainer(); + _cloudBlobContainer.Verify(x => x.DeleteIfExistsAsync(), Times.Once); + _cloudBlobContainer.Verify(x => x.CreateAsync(), Times.Never); + _cloudBlobContainer.Verify(x => x.CreateIfNotExistAsync(), Times.Never); + _cloudBlobContainer.Verify(x => x.SetPermissionsAsync(It.IsAny()), Times.Never); + } + } + + public abstract class BaseFacts + { + protected readonly Mock _cloudBlobClient; + protected readonly Mock _cloudBlobContainer; + protected readonly Mock> _options; + protected readonly AzureSearchJobConfiguration _config; + protected readonly RecordingLogger _logger; + protected readonly TimeSpan _retryDuration; + protected readonly BlobContainerBuilder _target; + + public BaseFacts(ITestOutputHelper output) + { + _cloudBlobClient = new Mock(); + _cloudBlobContainer = new Mock(); + _options = new Mock>(); + _config = new AzureSearchJobConfiguration + { + StorageContainer = "container-name", + }; + _logger = output.GetLogger(); + _retryDuration = TimeSpan.FromMilliseconds(10); + + _options + .Setup(x => x.Value) + .Returns(() => _config); + _cloudBlobClient + .Setup(x => x.GetContainerReference(It.IsAny())) + .Returns(() => _cloudBlobContainer.Object); + + _target = new BlobContainerBuilder( + _cloudBlobClient.Object, + _options.Object, + _logger, + _retryDuration); + } + + protected void EnableConflict() + { + _cloudBlobContainer + .SetupSequence(x => x.CreateAsync()) + .Throws(new StorageException( + new RequestResult + { + HttpStatusCode = (int)HttpStatusCode.Conflict, + }, + "Conflict.", + inner: null)) + .Returns(Task.CompletedTask); + } + + protected void VerifySetPermissions() + { + _cloudBlobContainer.Verify( + x => x.SetPermissionsAsync(It.Is(p => p.PublicAccess == BlobContainerPublicAccessType.Blob)), + Times.Once); + _cloudBlobContainer.Verify(x => x.SetPermissionsAsync(It.IsAny()), Times.Once); + } + + protected void VerifyGetContainer() + { + _cloudBlobClient.Verify(x => x.GetContainerReference(_config.StorageContainer), Times.Once); + _cloudBlobClient.Verify(x => x.GetContainerReference(It.IsAny()), Times.Once); + } + } + } + + public class IndexBuilderFacts + { + public class CreateSearchIndexAsync : BaseFacts + { + public CreateSearchIndexAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task CreatesIndex() + { + await _target.CreateSearchIndexAsync(); + + _indexesOperations.Verify( + x => x.CreateAsync(It.Is(y => y.Name == _config.SearchIndexName)), + Times.Once); + _indexesOperations.Verify( + x => x.CreateAsync(It.IsAny()), + Times.Once); + } + + [Fact] + public async Task CreatesScoringProfile() + { + Index createdIndex = null; + _indexesOperations + .Setup(o => o.CreateAsync(It.IsAny())) + .Callback(index => createdIndex = index) + .Returns(() => Task.FromResult(createdIndex)); + + await _target.CreateSearchIndexAsync(); + + Assert.NotNull(createdIndex); + + var result = Assert.Single(createdIndex.ScoringProfiles); + Assert.Equal(DefaultScoringProfile.Name, result.Name); + + // Verify field weights + Assert.Equal(2, result.TextWeights.Weights.Count); + + Assert.Contains(IndexFields.PackageId, result.TextWeights.Weights.Keys); + Assert.Equal(3.0, result.TextWeights.Weights[IndexFields.PackageId]); + + Assert.Contains(IndexFields.TokenizedPackageId, result.TextWeights.Weights.Keys); + Assert.Equal(4.0, result.TextWeights.Weights[IndexFields.TokenizedPackageId]); + + // Verify boosting functions + Assert.Equal(1, result.Functions.Count); + var downloadsBoost = result + .Functions + .Where(f => f.FieldName == IndexFields.Search.DownloadScore) + .FirstOrDefault(); + + Assert.NotNull(downloadsBoost); + Assert.Equal(5.0, downloadsBoost.Boost); + } + + [Fact] + public async Task ThrowsOnInvalidFieldInConfig() + { + _config.Scoring.FieldWeights["WARGLE"] = 123.0; + + var exception = await Assert.ThrowsAsync(() => _target.CreateSearchIndexAsync()); + + Assert.Contains("Unknown field 'WARGLE'", exception.Message); + } + } + + public class CreateHijackIndexAsync : BaseFacts + { + public CreateHijackIndexAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task CreatesIndex() + { + await _target.CreateHijackIndexAsync(); + + _indexesOperations.Verify( + x => x.CreateAsync(It.Is(y => y.Name == _config.HijackIndexName)), + Times.Once); + _indexesOperations.Verify( + x => x.CreateAsync(It.IsAny()), + Times.Once); + } + + [Fact] + public async Task DoesNotCreateScoringProfile() + { + Index createdIndex = null; + _indexesOperations + .Setup(o => o.CreateAsync(It.IsAny())) + .Callback(index => createdIndex = index) + .Returns(() => Task.FromResult(createdIndex)); + + await _target.CreateHijackIndexAsync(); + + Assert.NotNull(createdIndex); + Assert.Null(createdIndex.ScoringProfiles); + } + } + + public class CreateSearchIndexIfNotExistsAsync : BaseFacts + { + public CreateSearchIndexIfNotExistsAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task CreatesIndexIfNotExists() + { + _indexesOperations + .Setup(x => x.ExistsAsync(_config.SearchIndexName)) + .ReturnsAsync(false); + + await _target.CreateSearchIndexIfNotExistsAsync(); + + _indexesOperations.Verify( + x => x.CreateAsync(It.Is(y => y.Name == _config.SearchIndexName)), + Times.Once); + _indexesOperations.Verify( + x => x.CreateAsync(It.IsAny()), + Times.Once); + } + + [Fact] + public async Task DoesNotCreateIndexIfExists() + { + _indexesOperations + .Setup(x => x.ExistsAsync(_config.SearchIndexName)) + .ReturnsAsync(true); + + await _target.CreateSearchIndexIfNotExistsAsync(); + + _indexesOperations.Verify( + x => x.CreateAsync(It.IsAny()), + Times.Never); + } + } + + public class CreateHijackIndexIfNotExistsAsync : BaseFacts + { + public CreateHijackIndexIfNotExistsAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task CreatesIndexIfNotExists() + { + _indexesOperations + .Setup(x => x.ExistsAsync(_config.HijackIndexName)) + .ReturnsAsync(false); + + await _target.CreateHijackIndexIfNotExistsAsync(); + + _indexesOperations.Verify( + x => x.CreateAsync(It.Is(y => y.Name == _config.HijackIndexName)), + Times.Once); + _indexesOperations.Verify( + x => x.CreateAsync(It.IsAny()), + Times.Once); + } + + [Fact] + public async Task DoesNotCreateIndexIfExists() + { + _indexesOperations + .Setup(x => x.ExistsAsync(_config.HijackIndexName)) + .ReturnsAsync(true); + + await _target.CreateHijackIndexIfNotExistsAsync(); + + _indexesOperations.Verify( + x => x.CreateAsync(It.IsAny()), + Times.Never); + } + } + + public class DeleteSearchIndexIfExistsAsync : BaseFacts + { + public DeleteSearchIndexIfExistsAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task DoesNotDeleteIndexIfNotExists() + { + _indexesOperations + .Setup(x => x.ExistsAsync(_config.SearchIndexName)) + .ReturnsAsync(false); + + await _target.CreateSearchIndexIfNotExistsAsync(); + + _indexesOperations.Verify( + x => x.DeleteAsync(It.IsAny()), + Times.Never); + } + + [Fact] + public async Task DeletesIndexIfExists() + { + _indexesOperations + .Setup(x => x.ExistsAsync(_config.SearchIndexName)) + .ReturnsAsync(true); + + await _target.DeleteSearchIndexIfExistsAsync(); + + _indexesOperations.Verify( + x => x.DeleteAsync(_config.SearchIndexName), + Times.Once); + _indexesOperations.Verify( + x => x.DeleteAsync(It.IsAny()), + Times.Once); + } + } + + public class DeleteHijackIndexIfExistsAsync : BaseFacts + { + public DeleteHijackIndexIfExistsAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task DoesNotDeleteIndexIfNotExists() + { + _indexesOperations + .Setup(x => x.ExistsAsync(_config.HijackIndexName)) + .ReturnsAsync(false); + + await _target.CreateHijackIndexIfNotExistsAsync(); + + _indexesOperations.Verify( + x => x.DeleteAsync(It.IsAny()), + Times.Never); + } + + [Fact] + public async Task DeletesIndexIfExists() + { + _indexesOperations + .Setup(x => x.ExistsAsync(_config.HijackIndexName)) + .ReturnsAsync(true); + + await _target.DeleteHijackIndexIfExistsAsync(); + + _indexesOperations.Verify( + x => x.DeleteAsync(_config.HijackIndexName), + Times.Once); + _indexesOperations.Verify( + x => x.DeleteAsync(It.IsAny()), + Times.Once); + } + } + + public abstract class BaseFacts + { + protected readonly Mock _serviceClient; + protected readonly Mock _indexesOperations; + protected readonly Mock> _options; + protected readonly AzureSearchJobConfiguration _config; + protected readonly RecordingLogger _logger; + protected readonly IndexBuilder _target; + + public BaseFacts(ITestOutputHelper output) + { + _serviceClient = new Mock(); + _indexesOperations = new Mock(); + _options = new Mock>(); + _config = new AzureSearchJobConfiguration + { + SearchIndexName = "search", + HijackIndexName = "hijack", + + Scoring = new AzureSearchScoringConfiguration + { + FieldWeights = new Dictionary + { + { nameof(IndexFields.PackageId), 3.0 }, + { nameof(IndexFields.TokenizedPackageId), 4.0 }, + }, + + DownloadScoreBoost = 5.0, + } + }; + _logger = output.GetLogger(); + + _options + .Setup(x => x.Value) + .Returns(() => _config); + _serviceClient + .Setup(x => x.Indexes) + .Returns(() => _indexesOperations.Object); + + _target = new IndexBuilder( + _serviceClient.Object, + _options.Object, + _logger); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Models/CommittedDocumentFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Models/CommittedDocumentFacts.cs new file mode 100644 index 000000000..8d6018778 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Models/CommittedDocumentFacts.cs @@ -0,0 +1,62 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Xunit; + +namespace NuGet.Services.AzureSearch +{ + public class CurrentTimestampFacts + { + [Fact] + public void CapturesCurrentTimestampOnNextRead() + { + var target = new CurrentTimestamp(); + + target.SetOnNextRead(); + var before = DateTimeOffset.UtcNow; + var actual = target.Value; + var after = DateTimeOffset.UtcNow; + + Assert.NotNull(actual); + Assert.InRange(actual.Value, before, after); + } + + [Fact] + public void RetainsValueAfterFirstRead() + { + var target = new CurrentTimestamp(); + target.SetOnNextRead(); + var initial = target.Value; + + var actual = target.Value; + + Assert.NotNull(actual); + Assert.Equal(initial, actual); + } + + [Fact] + public void ReplacesSetValueWhenFlagIsSet() + { + var target = new CurrentTimestamp(); + target.SetOnNextRead(); + target.Value = DateTimeOffset.MaxValue; + + var actual = target.Value; + + Assert.NotNull(actual); + Assert.NotEqual(DateTimeOffset.MaxValue, actual); + } + + [Fact] + public void DoesNotReplaceSetValueWhenFlagIsNotSet() + { + var target = new CurrentTimestamp(); + target.Value = DateTimeOffset.MaxValue; + + var actual = target.Value; + + Assert.Equal(DateTimeOffset.MaxValue, actual); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/NuGet.Services.AzureSearch.Tests.csproj b/tests/NuGet.Services.AzureSearch.Tests/NuGet.Services.AzureSearch.Tests.csproj new file mode 100644 index 000000000..b55fefaa8 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/NuGet.Services.AzureSearch.Tests.csproj @@ -0,0 +1,144 @@ + + + + + Debug + AnyCPU + {6A9C3802-A2A2-49CF-87BD-C1303533B846} + Library + Properties + NuGet.Services.AzureSearch + NuGet.Services.AzureSearch.Tests + v4.7.2 + 512 + true + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {E97F23B8-ECB0-4AFA-B00C-015C39395FEF} + NuGet.Services.Metadata.Catalog + + + {4B4B1EFB-8F33-42E6-B79F-54E7F3293D31} + NuGet.Jobs.Common + + + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} + NuGet.Protocol.Catalog + + + {1a53fe3d-8041-4773-942f-d73aef5b82b2} + NuGet.Services.AzureSearch + + + {c3f9a738-9759-4b2b-a50d-6507b28a659b} + NuGet.Services.V3 + + + {ccb4d5ef-ac84-449d-ac6e-0a0ad295483a} + NuGet.Services.V3.Tests + + + + + 4.10.1 + + + 2.4.1 + + + 2.4.1 + runtime; build; native; contentfiles; analyzers + all + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + + + \ No newline at end of file diff --git a/tests/NuGet.Services.AzureSearch.Tests/Properties/AssemblyInfo.cs b/tests/NuGet.Services.AzureSearch.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..a0b0a7c8b --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,13 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Services.AzureSearch.Tests")] +[assembly: ComVisible(false)] +[assembly: Guid("6a9c3802-a2a2-49cf-87bd-c1303533b846")] + +[assembly: AssemblyMetadata("BuildDateUtc", "This is a fake build date for testing.")] +[assembly: AssemblyMetadata("CommitId", "This is a fake commit ID for testing.")] +[assembly: AssemblyInformationalVersion("1.0.0-fakefortesting")] diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchDocumentBuilderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchDocumentBuilderFacts.cs new file mode 100644 index 000000000..4589b6380 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchDocumentBuilderFacts.cs @@ -0,0 +1,1026 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Services.AzureSearch.Support; +using NuGet.Services.Entities; +using NuGet.Versioning; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch +{ + public class SearchDocumentBuilderFacts + { + public class LatestFlagsOrNull : BaseFacts + { + public LatestFlagsOrNull(ITestOutputHelper output) : base(output) + { + } + + [Theory] + [InlineData(SearchFilters.Default)] + [InlineData(SearchFilters.IncludeSemVer2)] + public void ExcludePrereleaseWithOnlyOnePrereleaseVersion(SearchFilters searchFilters) + { + var versionLists = VersionLists("1.0.0-alpha"); + + var actual = _target.LatestFlagsOrNull(versionLists, searchFilters); + + Assert.Null(actual); + } + + [Theory] + [InlineData(SearchFilters.Default)] + [InlineData(SearchFilters.IncludePrerelease)] + public void ExcludingSemVer2WithOnlySemVer2(SearchFilters searchFilters) + { + var versionLists = VersionLists("1.0.0+git", "2.0.0-alpha.1"); + + var actual = _target.LatestFlagsOrNull(versionLists, searchFilters); + + Assert.Null(actual); + } + + [Theory] + [InlineData(SearchFilters.IncludePrerelease)] + [InlineData(SearchFilters.IncludePrereleaseAndSemVer2)] + public void IncludePrereleaseWithOnlyOnePrereleaseVersion(SearchFilters searchFilters) + { + var versionLists = VersionLists("1.0.0-alpha"); + + var actual = _target.LatestFlagsOrNull(versionLists, searchFilters); + + Assert.Equal("1.0.0-alpha", actual.LatestVersionInfo.FullVersion); + Assert.False(actual.IsLatestStable); + Assert.True(actual.IsLatest); + } + + [Theory] + [InlineData(SearchFilters.Default)] + [InlineData(SearchFilters.IncludePrerelease)] + [InlineData(SearchFilters.IncludeSemVer2)] + [InlineData(SearchFilters.IncludePrereleaseAndSemVer2)] + public void OnlyOneStableVersion(SearchFilters searchFilters) + { + var versionLists = VersionLists("1.0.0"); + + var actual = _target.LatestFlagsOrNull(versionLists, searchFilters); + + Assert.Equal("1.0.0", actual.LatestVersionInfo.FullVersion); + Assert.True(actual.IsLatestStable); + Assert.True(actual.IsLatest); + } + + [Theory] + [InlineData(SearchFilters.Default, "1.0.0", true, false)] + [InlineData(SearchFilters.IncludeSemVer2, "1.0.0", true, false)] + [InlineData(SearchFilters.IncludePrerelease, "2.0.0-alpha", false, true)] + [InlineData(SearchFilters.IncludePrereleaseAndSemVer2, "2.0.0-alpha", false, true)] + public void LatestIsPrereleaseWithLowerStable(SearchFilters searchFilters, string latest, bool isLatestStable, bool isLatest) + { + var versionLists = VersionLists("1.0.0", "2.0.0-alpha"); + + var actual = _target.LatestFlagsOrNull(versionLists, searchFilters); + + Assert.Equal(latest, actual.LatestVersionInfo.FullVersion); + Assert.Equal(isLatestStable, actual.IsLatestStable); + Assert.Equal(isLatest, actual.IsLatest); + } + + [Theory] + [InlineData(SearchFilters.Default, "1.0.0", true, false)] + [InlineData(SearchFilters.IncludePrerelease, "2.0.0-alpha", false, true)] + [InlineData(SearchFilters.IncludeSemVer2, "3.0.0+git", true, false)] + [InlineData(SearchFilters.IncludePrereleaseAndSemVer2, "4.0.0-beta.1", false, true)] + public void AllVersionTypes(SearchFilters searchFilters, string latest, bool isLatestStable, bool isLatest) + { + var versionLists = VersionLists("1.0.0", "2.0.0-alpha", "3.0.0+git", "4.0.0-beta.1"); + + var actual = _target.LatestFlagsOrNull(versionLists, searchFilters); + + Assert.Equal(latest, actual.LatestVersionInfo.FullVersion); + Assert.Equal(isLatestStable, actual.IsLatestStable); + Assert.Equal(isLatest, actual.IsLatest); + } + + private static VersionLists VersionLists(params string[] versions) + { + return new VersionLists(new VersionListData(versions + .Select(x => NuGetVersion.Parse(x)) + .ToDictionary(x => x.ToFullString(), x => new VersionPropertiesData(listed: true, semVer2: x.IsSemVer2)))); + } + } + + public class Keyed : BaseFacts + { + public Keyed(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task SetsExpectedProperties() + { + var document = _target.Keyed(Data.PackageId, Data.SearchFilters); + + var json = await SerializationUtilities.SerializeToJsonAsync(document); + Assert.Equal(@"{ + ""value"": [ + { + ""@search.action"": ""upload"", + ""key"": ""windowsazure_storage-d2luZG93c2F6dXJlLnN0b3JhZ2U1-IncludePrereleaseAndSemVer2"" + } + ] +}", json); + } + } + + public class UpdateOwners : BaseFacts + { + public UpdateOwners(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task SetsExpectedProperties() + { + var document = _target.UpdateOwners( + Data.PackageId, + Data.SearchFilters, + Data.Owners); + + SetDocumentLastUpdated(document); + var json = await SerializationUtilities.SerializeToJsonAsync(document); + Assert.Equal(@"{ + ""value"": [ + { + ""@search.action"": ""upload"", + ""owners"": [ + ""Microsoft"", + ""azure-sdk"" + ], + ""lastUpdatedDocument"": ""2018-12-14T09:30:00+00:00"", + ""lastDocumentType"": ""NuGet.Services.AzureSearch.SearchDocument+UpdateOwners"", + ""lastUpdatedFromCatalog"": false, + ""key"": ""windowsazure_storage-d2luZG93c2F6dXJlLnN0b3JhZ2U1-IncludePrereleaseAndSemVer2"" + } + ] +}", json); + } + } + + public class UpdateDownloadCount : BaseFacts + { + public UpdateDownloadCount(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task SetsExpectedProperties() + { + var document = _target.UpdateDownloadCount( + Data.PackageId, + Data.SearchFilters, + Data.TotalDownloadCount); + + SetDocumentLastUpdated(document); + var json = await SerializationUtilities.SerializeToJsonAsync(document); + Assert.Equal(@"{ + ""value"": [ + { + ""@search.action"": ""upload"", + ""totalDownloadCount"": 1001, + ""downloadScore"": 0.14381174563233068, + ""lastUpdatedDocument"": ""2018-12-14T09:30:00+00:00"", + ""lastDocumentType"": ""NuGet.Services.AzureSearch.SearchDocument+UpdateDownloadCount"", + ""lastUpdatedFromCatalog"": false, + ""key"": ""windowsazure_storage-d2luZG93c2F6dXJlLnN0b3JhZ2U1-IncludePrereleaseAndSemVer2"" + } + ] +}", json); + } + } + + public class UpdateVersionListFromCatalog : BaseFacts + { + public UpdateVersionListFromCatalog(ITestOutputHelper output) : base(output) + { + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task SetsExpectedProperties(bool isLatestStable, bool isLatest) + { + var document = _target.UpdateVersionListFromCatalog( + Data.PackageId, + Data.SearchFilters, + Data.CommitTimestamp, + Data.CommitId, + Data.Versions, + isLatestStable, + isLatest); + + SetDocumentLastUpdated(document); + var json = await SerializationUtilities.SerializeToJsonAsync(document); + Assert.Equal(@"{ + ""value"": [ + { + ""@search.action"": ""upload"", + ""versions"": [ + ""1.0.0"", + ""2.0.0+git"", + ""3.0.0-alpha.1"", + ""7.1.2-alpha+git"" + ], + ""isLatestStable"": " + isLatestStable.ToString().ToLowerInvariant() + @", + ""isLatest"": " + isLatest.ToString().ToLowerInvariant() + @", + ""lastCommitTimestamp"": ""2018-12-13T12:30:00+00:00"", + ""lastCommitId"": ""6b9b24dd-7aec-48ae-afc1-2a117e3d50d1"", + ""lastUpdatedDocument"": ""2018-12-14T09:30:00+00:00"", + ""lastDocumentType"": ""NuGet.Services.AzureSearch.SearchDocument+UpdateVersionList"", + ""lastUpdatedFromCatalog"": true, + ""key"": ""windowsazure_storage-d2luZG93c2F6dXJlLnN0b3JhZ2U1-IncludePrereleaseAndSemVer2"" + } + ] +}", json); + } + } + + public class UpdateVersionListAndOwnersFromCatalog : BaseFacts + { + public UpdateVersionListAndOwnersFromCatalog(ITestOutputHelper output) : base(output) + { + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public async Task SetsExpectedProperties(bool isLatestStable, bool isLatest) + { + var document = _target.UpdateVersionListAndOwnersFromCatalog( + Data.PackageId, + Data.SearchFilters, + Data.CommitTimestamp, + Data.CommitId, + Data.Versions, + isLatestStable, + isLatest, + Data.Owners); + + SetDocumentLastUpdated(document); + var json = await SerializationUtilities.SerializeToJsonAsync(document); + Assert.Equal(@"{ + ""value"": [ + { + ""@search.action"": ""upload"", + ""owners"": [ + ""Microsoft"", + ""azure-sdk"" + ], + ""versions"": [ + ""1.0.0"", + ""2.0.0+git"", + ""3.0.0-alpha.1"", + ""7.1.2-alpha+git"" + ], + ""isLatestStable"": " + isLatestStable.ToString().ToLowerInvariant() + @", + ""isLatest"": " + isLatest.ToString().ToLowerInvariant() + @", + ""lastCommitTimestamp"": ""2018-12-13T12:30:00+00:00"", + ""lastCommitId"": ""6b9b24dd-7aec-48ae-afc1-2a117e3d50d1"", + ""lastUpdatedDocument"": ""2018-12-14T09:30:00+00:00"", + ""lastDocumentType"": ""NuGet.Services.AzureSearch.SearchDocument+UpdateVersionListAndOwners"", + ""lastUpdatedFromCatalog"": true, + ""key"": ""windowsazure_storage-d2luZG93c2F6dXJlLnN0b3JhZ2U1-IncludePrereleaseAndSemVer2"" + } + ] +}", json); + } + } + + public class UpdateLatestFromCatalog : BaseFacts + { + public UpdateLatestFromCatalog(ITestOutputHelper output) : base(output) + { + } + + [Theory] + [MemberData(nameof(MissingTitles))] + public void UsesIdWhenMissingForTitle(string title) + { + var leaf = Data.Leaf; + leaf.Title = title; + + var document = _target.UpdateLatestFromCatalog( + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + normalizedVersion: Data.NormalizedVersion, + fullVersion: Data.FullVersion, + leaf: leaf, + owners: Data.Owners); + + Assert.Equal(Data.PackageId, document.Title); + } + + [Theory] + [MemberData(nameof(MissingTitles))] + public void UsesLowerIdWhenMissingForSortableTitle(string title) + { + var leaf = Data.Leaf; + leaf.Title = title; + + var document = _target.UpdateLatestFromCatalog( + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + normalizedVersion: Data.NormalizedVersion, + fullVersion: Data.FullVersion, + leaf: leaf, + owners: Data.Owners); + + Assert.Equal(Data.PackageId.ToLowerInvariant(), document.SortableTitle); + } + + [Theory] + [MemberData(nameof(AllSearchFilters))] + public async Task SetsExpectedProperties(SearchFilters searchFilters, string expected) + { + var document = _target.UpdateLatestFromCatalog( + searchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + normalizedVersion: Data.NormalizedVersion, + fullVersion: Data.FullVersion, + leaf: Data.Leaf, + owners: Data.Owners); + + SetDocumentLastUpdated(document); + var json = await SerializationUtilities.SerializeToJsonAsync(document); + Assert.Equal(@"{ + ""value"": [ + { + ""@search.action"": ""upload"", + ""owners"": [ + ""Microsoft"", + ""azure-sdk"" + ], + ""searchFilters"": """ + expected + @""", + ""filterablePackageTypes"": [ + ""dependency"" + ], + ""fullVersion"": ""7.1.2-alpha+git"", + ""versions"": [ + ""1.0.0"", + ""2.0.0+git"", + ""3.0.0-alpha.1"", + ""7.1.2-alpha+git"" + ], + ""packageTypes"": [ + ""Dependency"" + ], + ""isLatestStable"": false, + ""isLatest"": true, + ""semVerLevel"": 2, + ""authors"": ""Microsoft"", + ""copyright"": ""© Microsoft Corporation. All rights reserved."", + ""created"": ""2017-01-01T00:00:00+00:00"", + ""description"": ""Description."", + ""fileSize"": 3039254, + ""flattenedDependencies"": ""Microsoft.Data.OData:5.6.4:net40-client|Newtonsoft.Json:6.0.8:net40-client"", + ""hash"": ""oMs9XKzRTsbnIpITcqZ5XAv1h2z6oyJ33+Z/PJx36iVikge/8wm5AORqAv7soKND3v5/0QWW9PQ0ktQuQu9aQQ=="", + ""hashAlgorithm"": ""SHA512"", + ""iconUrl"": ""http://go.microsoft.com/fwlink/?LinkID=288890"", + ""language"": ""en-US"", + ""lastEdited"": ""2017-01-02T00:00:00+00:00"", + ""licenseUrl"": ""http://go.microsoft.com/fwlink/?LinkId=331471"", + ""minClientVersion"": ""2.12"", + ""normalizedVersion"": ""7.1.2-alpha"", + ""originalVersion"": ""7.1.2.0-alpha+git"", + ""packageId"": ""WindowsAzure.Storage"", + ""prerelease"": true, + ""projectUrl"": ""https://github.com/Azure/azure-storage-net"", + ""published"": ""2017-01-03T00:00:00+00:00"", + ""releaseNotes"": ""Release notes."", + ""requiresLicenseAcceptance"": true, + ""sortableTitle"": ""windows azure storage"", + ""summary"": ""Summary."", + ""tags"": [ + ""Microsoft"", + ""Azure"", + ""Storage"", + ""Table"", + ""Blob"", + ""File"", + ""Queue"", + ""Scalable"", + ""windowsazureofficial"" + ], + ""title"": ""Windows Azure Storage"", + ""tokenizedPackageId"": ""WindowsAzure.Storage"", + ""lastCommitTimestamp"": ""2018-12-13T12:30:00+00:00"", + ""lastCommitId"": ""6b9b24dd-7aec-48ae-afc1-2a117e3d50d1"", + ""lastUpdatedDocument"": ""2018-12-14T09:30:00+00:00"", + ""lastDocumentType"": ""NuGet.Services.AzureSearch.SearchDocument+UpdateLatest"", + ""lastUpdatedFromCatalog"": true, + ""key"": ""windowsazure_storage-d2luZG93c2F6dXJlLnN0b3JhZ2U1-" + expected + @""" + } + ] +}", json); + } + + [Theory] + [MemberData(nameof(CatalogPackageTypesData))] + public void SetsExpectedPackageTypes(List packageTypes, string[] expectedFilterable, string[] expectedDisplay) + { + var leaf = Data.Leaf; + leaf.PackageTypes = packageTypes; + + var document = _target.UpdateLatestFromCatalog( + SearchFilters.Default, + Data.Versions, + isLatestStable: false, + isLatest: true, + normalizedVersion: Data.NormalizedVersion, + fullVersion: Data.FullVersion, + leaf: leaf, + owners: Data.Owners); + + SetDocumentLastUpdated(document); + Assert.Equal(document.FilterablePackageTypes.Length, document.PackageTypes.Length); + Assert.Equal(expectedFilterable, document.FilterablePackageTypes); + Assert.Equal(expectedDisplay, document.PackageTypes); + } + + [Fact] + public void LeavesNullRequiresLicenseAcceptanceAsNull() + { + var leaf = Data.Leaf; + leaf.RequireLicenseAcceptance = null; + + var document = _target.UpdateLatestFromCatalog( + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + normalizedVersion: Data.NormalizedVersion, + fullVersion: Data.FullVersion, + leaf: leaf, + owners: Data.Owners); + + Assert.Null(document.RequiresLicenseAcceptance); + } + + [Fact] + public void SetsLicenseUrlToGalleryWhenPackageHasLicenseExpression() + { + var leaf = Data.Leaf; + leaf.LicenseExpression = "MIT"; + + var document = _target.UpdateLatestFromCatalog( + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + normalizedVersion: Data.NormalizedVersion, + fullVersion: Data.FullVersion, + leaf: leaf, + owners: Data.Owners); + + Assert.Equal(Data.GalleryLicenseUrl, document.LicenseUrl); + } + + [Fact] + public void SetsLicenseUrlToGalleryWhenPackageHasLicenseFile() + { + var leaf = Data.Leaf; + leaf.LicenseFile = "LICENSE.txt"; + + var document = _target.UpdateLatestFromCatalog( + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + normalizedVersion: Data.NormalizedVersion, + fullVersion: Data.FullVersion, + leaf: leaf, + owners: Data.Owners); + + Assert.Equal(Data.GalleryLicenseUrl, document.LicenseUrl); + } + + [Fact] + public void SetsIconUrlToFlatContainerWhenPackageHasIconFileAndIconUrl() + { + var leaf = Data.Leaf; + leaf.IconUrl = "https://other-example/icon.png"; + leaf.IconFile = "icon.png"; + + var document = _target.UpdateLatestFromCatalog( + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + normalizedVersion: Data.NormalizedVersion, + fullVersion: Data.FullVersion, + leaf: leaf, + owners: Data.Owners); + + Assert.Equal(Data.FlatContainerIconUrl, document.IconUrl); + } + + [Fact] + public void SetsIconUrlToFlatContainerWhenPackageHasIconFileAndNoIconUrl() + { + var leaf = Data.Leaf; + leaf.IconUrl = null; + leaf.IconFile = "icon.png"; + + var document = _target.UpdateLatestFromCatalog( + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + normalizedVersion: Data.NormalizedVersion, + fullVersion: Data.FullVersion, + leaf: leaf, + owners: Data.Owners); + + Assert.Equal(Data.FlatContainerIconUrl, document.IconUrl); + } + } + + public class FullFromDb : BaseFacts + { + public FullFromDb(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public void NormalizesSortableTitle() + { + var package = Data.PackageEntity; + package.Title = " Some Title "; + + var document = _target.FullFromDb( + Data.PackageId, + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + fullVersion: Data.FullVersion, + package: package, + owners: Data.Owners, + totalDownloadCount: Data.TotalDownloadCount, + isExcludedByDefault: false); + + Assert.Equal("some title", document.SortableTitle); + } + + [Theory] + [MemberData(nameof(MissingTitles))] + public void UsesIdWhenMissingForTitle(string title) + { + var package = Data.PackageEntity; + package.Title = title; + + var document = _target.FullFromDb( + Data.PackageId, + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + fullVersion: Data.FullVersion, + package: package, + owners: Data.Owners, + totalDownloadCount: Data.TotalDownloadCount, + isExcludedByDefault: false); + + Assert.Equal(Data.PackageId, document.Title); + } + + [Theory] + [MemberData(nameof(MissingTitles))] + public void UsesLowerIdWhenMissingForSortableTitle(string title) + { + var package = Data.PackageEntity; + package.Title = title; + + var document = _target.FullFromDb( + Data.PackageId, + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + fullVersion: Data.FullVersion, + package: package, + owners: Data.Owners, + totalDownloadCount: Data.TotalDownloadCount, + isExcludedByDefault: false); + + Assert.Equal(Data.PackageId.ToLowerInvariant(), document.SortableTitle); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void SetsIsExcludedByDefaultPropertyCorrectly(bool shouldBeExcluded) + { + var document = _target.FullFromDb( + Data.PackageId, + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + fullVersion: Data.FullVersion, + package: Data.PackageEntity, + owners: Data.Owners, + totalDownloadCount: Data.TotalDownloadCount, + isExcludedByDefault: shouldBeExcluded); + + Assert.Equal(shouldBeExcluded, document.IsExcludedByDefault); + } + + [Fact] + public async Task SerializesNullSemVerLevel() + { + var package = Data.PackageEntity; + package.SemVerLevelKey = SemVerLevelKey.Unknown; + + var document = _target.FullFromDb( + Data.PackageId, + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + fullVersion: Data.FullVersion, + package: package, + owners: Data.Owners, + totalDownloadCount: Data.TotalDownloadCount, + isExcludedByDefault: false); + + var json = await SerializationUtilities.SerializeToJsonAsync(document); + Assert.Contains("\"semVerLevel\": null,", json); + } + + [Theory] + [MemberData(nameof(AllSearchFilters))] + public async Task SetsExpectedProperties(SearchFilters searchFilters, string expected) + { + var document = _target.FullFromDb( + Data.PackageId, + searchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + fullVersion: Data.FullVersion, + package: Data.PackageEntity, + owners: Data.Owners, + totalDownloadCount: Data.TotalDownloadCount, + isExcludedByDefault: false); + + SetDocumentLastUpdated(document); + var json = await SerializationUtilities.SerializeToJsonAsync(document); + Assert.Equal(@"{ + ""value"": [ + { + ""@search.action"": ""upload"", + ""totalDownloadCount"": 1001, + ""downloadScore"": 0.14381174563233068, + ""isExcludedByDefault"": false, + ""owners"": [ + ""Microsoft"", + ""azure-sdk"" + ], + ""searchFilters"": """ + expected + @""", + ""filterablePackageTypes"": [ + ""dependency"" + ], + ""fullVersion"": ""7.1.2-alpha+git"", + ""versions"": [ + ""1.0.0"", + ""2.0.0+git"", + ""3.0.0-alpha.1"", + ""7.1.2-alpha+git"" + ], + ""packageTypes"": [ + ""Dependency"" + ], + ""isLatestStable"": false, + ""isLatest"": true, + ""semVerLevel"": 2, + ""authors"": ""Microsoft"", + ""copyright"": ""© Microsoft Corporation. All rights reserved."", + ""created"": ""2017-01-01T00:00:00+00:00"", + ""description"": ""Description."", + ""fileSize"": 3039254, + ""flattenedDependencies"": ""Microsoft.Data.OData:5.6.4:net40-client|Newtonsoft.Json:6.0.8:net40-client"", + ""hash"": ""oMs9XKzRTsbnIpITcqZ5XAv1h2z6oyJ33+Z/PJx36iVikge/8wm5AORqAv7soKND3v5/0QWW9PQ0ktQuQu9aQQ=="", + ""hashAlgorithm"": ""SHA512"", + ""iconUrl"": ""http://go.microsoft.com/fwlink/?LinkID=288890"", + ""language"": ""en-US"", + ""lastEdited"": ""2017-01-02T00:00:00+00:00"", + ""licenseUrl"": ""http://go.microsoft.com/fwlink/?LinkId=331471"", + ""minClientVersion"": ""2.12"", + ""normalizedVersion"": ""7.1.2-alpha"", + ""originalVersion"": ""7.1.2.0-alpha+git"", + ""packageId"": ""WindowsAzure.Storage"", + ""prerelease"": true, + ""projectUrl"": ""https://github.com/Azure/azure-storage-net"", + ""published"": ""2017-01-03T00:00:00+00:00"", + ""releaseNotes"": ""Release notes."", + ""requiresLicenseAcceptance"": true, + ""sortableTitle"": ""windows azure storage"", + ""summary"": ""Summary."", + ""tags"": [ + ""Microsoft"", + ""Azure"", + ""Storage"", + ""Table"", + ""Blob"", + ""File"", + ""Queue"", + ""Scalable"", + ""windowsazureofficial"" + ], + ""title"": ""Windows Azure Storage"", + ""tokenizedPackageId"": ""WindowsAzure.Storage"", + ""lastCommitTimestamp"": null, + ""lastCommitId"": null, + ""lastUpdatedDocument"": ""2018-12-14T09:30:00+00:00"", + ""lastDocumentType"": ""NuGet.Services.AzureSearch.SearchDocument+Full"", + ""lastUpdatedFromCatalog"": false, + ""key"": ""windowsazure_storage-d2luZG93c2F6dXJlLnN0b3JhZ2U1-" + expected + @""" + } + ] +}", json); + } + + [Theory] + [MemberData(nameof(DBPackageTypesData))] + public void SetsExpectedPackageTypes(List packageTypes, string[] expectedFilterable, string[] expectedDisplay) + { + var package = Data.PackageEntity; + package.PackageTypes = packageTypes; + + var document = _target.FullFromDb( + Data.PackageId, + SearchFilters.Default, + Data.Versions, + isLatestStable: false, + isLatest: true, + fullVersion: Data.FullVersion, + package: package, + owners: Data.Owners, + totalDownloadCount: Data.TotalDownloadCount, + isExcludedByDefault: false); + + SetDocumentLastUpdated(document); + Assert.Equal(document.FilterablePackageTypes.Length, document.PackageTypes.Length); + Assert.Equal(expectedFilterable, document.FilterablePackageTypes); + Assert.Equal(expectedDisplay, document.PackageTypes); + } + + [Fact] + public void SplitsTags() + { + var package = Data.PackageEntity; + package.Tags = "foo; BAR | Baz"; + + var document = _target.FullFromDb( + Data.PackageId, + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + fullVersion: Data.FullVersion, + package: package, + owners: Data.Owners, + totalDownloadCount: Data.TotalDownloadCount, + isExcludedByDefault: false); + + Assert.Equal(new[] { "foo", "BAR", "Baz" }, document.Tags); + } + + [Fact] + public void SetsLicenseUrlToGalleryWhenPackageHasLicenseExpression() + { + var package = Data.PackageEntity; + package.LicenseExpression = "MIT"; + + var document = _target.FullFromDb( + Data.PackageId, + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + fullVersion: Data.FullVersion, + package: package, + owners: Data.Owners, + totalDownloadCount: Data.TotalDownloadCount, + isExcludedByDefault: false); + + Assert.Equal(Data.GalleryLicenseUrl, document.LicenseUrl); + } + + [Theory] + [InlineData(EmbeddedLicenseFileType.PlainText)] + [InlineData(EmbeddedLicenseFileType.Markdown)] + public void SetsLicenseUrlToGalleryWhenPackageHasLicenseFile(EmbeddedLicenseFileType type) + { + var package = Data.PackageEntity; + package.EmbeddedLicenseType = type; + + var document = _target.FullFromDb( + Data.PackageId, + Data.SearchFilters, + Data.Versions, + isLatestStable: false, + isLatest: true, + fullVersion: Data.FullVersion, + package: package, + owners: Data.Owners, + totalDownloadCount: Data.TotalDownloadCount, + isExcludedByDefault: false); + + Assert.Equal(Data.GalleryLicenseUrl, document.LicenseUrl); + } + } + + public abstract class BaseFacts + { + protected readonly ITestOutputHelper _output; + protected readonly Mock> _options; + protected readonly BaseDocumentBuilder _baseDocumentBuilder; + protected readonly AzureSearchJobConfiguration _config; + protected readonly SearchDocumentBuilder _target; + + public static IEnumerable MissingTitles = new[] + { + new object[] { null }, + new object[] { string.Empty }, + new object[] { " " }, + new object[] { " \t"}, + }; + + public static IEnumerable AllSearchFilters => new[] + { + new object[] { SearchFilters.Default, "Default" }, + new object[] { SearchFilters.IncludePrerelease, "IncludePrerelease" }, + new object[] { SearchFilters.IncludeSemVer2, "IncludeSemVer2" }, + new object[] { SearchFilters.IncludePrereleaseAndSemVer2, "IncludePrereleaseAndSemVer2" }, + }; + + public static IEnumerable CatalogPackageTypesData => new[] + { + new object[] { + new List { + new NuGet.Protocol.Catalog.PackageType + { + Name = "DotNetCliTool" + } + }, + new string[] { "dotnetclitool" }, + new string[] { "DotNetCliTool" } + }, + + new object[] { + null, + new string[] { "dependency" }, + new string[] { "Dependency" } + }, + + new object[] { + new List(), + new string[] { "dependency" }, + new string[] { "Dependency" } + }, + + new object[] { + new List { + new NuGet.Protocol.Catalog.PackageType + { + Name = "DotNetCliTool" + }, + new NuGet.Protocol.Catalog.PackageType + { + Name = "Dependency" + } + }, + new string[] { "dotnetclitool", "dependency" }, + new string[] { "DotNetCliTool", "Dependency" }, + }, + + new object[] { + new List { + new NuGet.Protocol.Catalog.PackageType + { + Name = "DotNetCliTool", + Version = "1.0.0" + } + }, + new string[] { "dotnetclitool" }, + new string[] { "DotNetCliTool" } + }, + }; + + public static IEnumerable DBPackageTypesData => new[] + { + new object[] { + new List { + new PackageType + { + Name = "DotNetCliTool" + } + }, + new string[] { "dotnetclitool" }, + new string[] { "DotNetCliTool" } + }, + + new object[] { + null, + new string[] { "dependency" }, + new string[] { "Dependency" } + }, + + new object[] { + new List(), + new string[] { "dependency" }, + new string[] { "Dependency" } + }, + + new object[] { + new List { + new PackageType + { + Name = "DotNetCliTool" + }, + new PackageType + { + Name = "Dependency" + } + }, + new string[] { "dotnetclitool", "dependency" }, + new string[] { "DotNetCliTool", "Dependency" }, + }, + + new object[] { + new List { + new PackageType + { + Name = "DotNetCliTool", + Version = "1.0.0" + } + }, + new string[] { "dotnetclitool" }, + new string[] { "DotNetCliTool" } + }, + }; + + [Fact] + public void AllSearchFiltersAreCovered() + { + var testedSearchFilters = AllSearchFilters.Select(x => (SearchFilters)x[0]).ToList(); + var allSearchFilters = Enum.GetValues(typeof(SearchFilters)).Cast().ToList(); + + Assert.Empty(testedSearchFilters.Except(allSearchFilters)); + Assert.Empty(allSearchFilters.Except(testedSearchFilters)); + } + + public void SetDocumentLastUpdated(IUpdatedDocument document) + { + Data.SetDocumentLastUpdated(document, _output); + } + + public BaseFacts(ITestOutputHelper output) + { + _output = output; + _options = new Mock>(); + _baseDocumentBuilder = new BaseDocumentBuilder(_options.Object); // We intentionally don't mock this. + _config = new AzureSearchJobConfiguration + { + GalleryBaseUrl = Data.GalleryBaseUrl, + FlatContainerBaseUrl = Data.FlatContainerBaseUrl, + FlatContainerContainerName = Data.FlatContainerContainerName, + }; + + _options.Setup(o => o.Value).Returns(() => _config); + + _target = new SearchDocumentBuilder(_baseDocumentBuilder); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchIndexActionBuilderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchIndexActionBuilderFacts.cs new file mode 100644 index 000000000..47dd17332 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchIndexActionBuilderFacts.cs @@ -0,0 +1,153 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using Moq; +using NuGet.Services.AzureSearch.Support; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch +{ + public class SearchIndexActionBuilderFacts + { + public class UpdateAsync : Facts + { + public UpdateAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task UpdatesSearchDocumentsWithVersionMatchingAllFilters() + { + VersionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary + { + { + "1.0.0", + new VersionPropertiesData(listed: true, semVer2: false) + }, + }), + AccessConditionWrapper.GenerateIfNotExistsCondition()); + + var indexActions = await Target.UpdateAsync(Data.PackageId, BuildDocument); + + Assert.Same(VersionListDataResult, indexActions.VersionListDataResult); + Assert.Empty(indexActions.Hijack); + + Assert.Equal(4, indexActions.Search.Count); + Assert.All(indexActions.Search, x => Assert.IsType(x.Document)); + Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.Merge, x.ActionType)); + + Assert.Single(indexActions.Search, x => x.Document.Key == SearchFilters.Default.ToString()); + Assert.Single(indexActions.Search, x => x.Document.Key == SearchFilters.IncludePrerelease.ToString()); + Assert.Single(indexActions.Search, x => x.Document.Key == SearchFilters.IncludeSemVer2.ToString()); + Assert.Single(indexActions.Search, x => x.Document.Key == SearchFilters.IncludePrereleaseAndSemVer2.ToString()); + } + + [Fact] + public async Task UpdatesSearchDocumentsWithVersionMatchingSomeFilters() + { + VersionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary + { + { + "1.0.0-beta", + new VersionPropertiesData(listed: true, semVer2: false) + }, + }), + AccessConditionWrapper.GenerateIfNotExistsCondition()); + + var indexActions = await Target.UpdateAsync(Data.PackageId, BuildDocument); + + Assert.Same(VersionListDataResult, indexActions.VersionListDataResult); + Assert.Empty(indexActions.Hijack); + + Assert.Equal(2, indexActions.Search.Count); + Assert.All(indexActions.Search, x => Assert.IsType(x.Document)); + Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.Merge, x.ActionType)); + + Assert.Single(indexActions.Search, x => x.Document.Key == SearchFilters.IncludePrerelease.ToString()); + Assert.Single(indexActions.Search, x => x.Document.Key == SearchFilters.IncludePrereleaseAndSemVer2.ToString()); + } + + [Fact] + public async Task UpdatesSearchDocumentsWithOnlyUnlistedVersions() + { + VersionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary + { + { + "1.0.0-beta", + new VersionPropertiesData(listed: false, semVer2: false) + }, + }), + AccessConditionWrapper.GenerateIfNotExistsCondition()); + + var indexActions = await Target.UpdateAsync(Data.PackageId, BuildDocument); + + Assert.Same(VersionListDataResult, indexActions.VersionListDataResult); + Assert.Empty(indexActions.Hijack); + Assert.Empty(indexActions.Search); + } + + [Fact] + public async Task UpdatesSearchDocumentsWithNoVersions() + { + VersionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + AccessConditionWrapper.GenerateIfNotExistsCondition()); + + var indexActions = await Target.UpdateAsync(Data.PackageId, BuildDocument); + + Assert.Same(VersionListDataResult, indexActions.VersionListDataResult); + Assert.Empty(indexActions.Hijack); + Assert.Empty(indexActions.Search); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + VersionListDataClient = new Mock(); + Search = new Mock(); + Logger = output.GetLogger(); + + VersionListDataResult = new ResultAndAccessCondition( + new VersionListData(new Dictionary()), + AccessConditionWrapper.GenerateIfNotExistsCondition()); + + VersionListDataClient + .Setup(x => x.ReadAsync(It.IsAny())) + .ReturnsAsync(() => VersionListDataResult); + Search + .Setup(x => x.UpdateOwners(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((_, sf, __) => new SearchDocument.UpdateOwners + { + Key = sf.ToString(), + }); + + Target = new SearchIndexActionBuilder(VersionListDataClient.Object, Logger); + } + + public Mock VersionListDataClient { get; } + public Mock Search { get; } + public RecordingLogger Logger { get; } + public ResultAndAccessCondition VersionListDataResult { get; set; } + public SearchIndexActionBuilder Target { get; } + + /// + /// The document is used as a simple example but other documents + /// could be produced. + /// + public SearchDocument.UpdateOwners BuildDocument(SearchFilters sf) + { + return Search.Object.UpdateOwners(Data.PackageId, sf, Data.Owners); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchService/AuxiliaryDataCacheFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchService/AuxiliaryDataCacheFacts.cs new file mode 100644 index 000000000..3c8647e91 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchService/AuxiliaryDataCacheFacts.cs @@ -0,0 +1,286 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Moq; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.AzureSearch.Support; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class AuxiliaryDataCacheFacts + { + public class EnsureInitialized : BaseFacts + { + public EnsureInitialized(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public void DefaultsToFalse() + { + Assert.False(_target.Initialized); + } + } + + public class InitializeAsync : BaseFacts + { + public InitializeAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task InitializesWhenUninitialized() + { + await _target.EnsureInitializedAsync(); + + Assert.True(_target.Initialized); + VerifyReadWithNoETag(); + var message = Assert.Single(_logger.Messages.Where(x => x.Contains("Done reloading auxiliary data."))); + Assert.EndsWith("Not modified: ", message); + Assert.Contains("Reloaded: Downloads, VerifiedPackages", message); + } + + [Fact] + public async Task DoesNotInitializeAgainWhenAlreadyInitialized() + { + // Arrange + await _target.EnsureInitializedAsync(); + _downloadDataClient.Invocations.Clear(); + _verifiedPackagesDataClient.Invocations.Clear(); + + // Act + await _target.EnsureInitializedAsync(); + + // Assert + Assert.True(_target.Initialized); + VerifyNoRead(); + } + + [Fact] + public async Task DoesNotInitializeAgainWhenCalledDuringInitialize() + { + // Arrange + var downloadsTcs = new TaskCompletionSource>(); + var startedDownloadTcs = new TaskCompletionSource(); + _downloadDataClient + .Setup(x => x.ReadLatestIndexedAsync(It.IsAny(), It.IsAny())) + .Returns(async () => + { + startedDownloadTcs.TrySetResult(true); + return await downloadsTcs.Task; + }); + var otherTask = _target.EnsureInitializedAsync(); + + // Act + var thisTask = _target.EnsureInitializedAsync(); + await startedDownloadTcs.Task; + downloadsTcs.TrySetResult(_downloadData); + await thisTask; + + // Assert + Assert.True(_target.Initialized); + VerifyReadWithNoETag(); + } + } + + public class TryLoadAsync : BaseFacts + { + public TryLoadAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task InitializesWhenUninitialized() + { + await _target.TryLoadAsync(_token); + + Assert.True(_target.Initialized); + VerifyReadWithNoETag(); + } + + [Fact] + public async Task InitializesAgainWhenAlreadyInitialized() + { + // Arrange + await _target.TryLoadAsync(_token); + _downloadDataClient.Invocations.Clear(); + _verifiedPackagesDataClient.Invocations.Clear(); + + // Act + await _target.TryLoadAsync(_token); + + // Assert + Assert.True(_target.Initialized); + VerifyReadWithETag(); + } + + [Fact] + public async Task ResetsStringCacheCounts() + { + // Perform two auxiliary file loads and verify the cache numbers emitted to telemetry. + var invocations = 0; + _downloadDataClient + .Setup(x => x.ReadLatestIndexedAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => _downloadData) + .Callback((_, sc) => + { + invocations++; + sc.Dedupe(new string('a', 1)); // 1: miss 2: hit + sc.Dedupe(new string('a', 1)); // 1: hit 2: hit + sc.Dedupe(new string('b', 1)); // 1: miss 2: hit + if (invocations > 1) + { + sc.Dedupe(new string('a', 1)); // 1: n/a 2: hit + sc.Dedupe(new string('d', 1)); // 1: n/a 2: miss + } + }); + _verifiedPackagesDataClient + .Setup(x => x.ReadLatestAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => _verifiedPackages) + .Callback((_, sc) => + { + sc.Dedupe(new string('a', 1)); // 1: hit 2: hit + sc.Dedupe(new string('b', 1)); // 1: hit 2: hit + sc.Dedupe(new string('c', 1)); // 1: miss 2: hit + sc.Dedupe(new string('c', 1)); // 1: miss 2: hit + }); + + await _target.TryLoadAsync(_token); + await _target.TryLoadAsync(_token); + + _telemetryService.Verify( + x => x.TrackAuxiliaryFilesStringCache(3, 3, 7, 4), + Times.Once); + _telemetryService.Verify( + x => x.TrackAuxiliaryFilesStringCache(4, 4, 9, 8), + Times.Once); + } + } + + public class Get : BaseFacts + { + public Get(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public void ThrowsWhenNotInitialized() + { + var ex = Assert.Throws(() => _target.Get()); + Assert.False(_target.Initialized); + Assert.Equal("The auxiliary data has not been loaded yet. Call LoadAsync.", ex.Message); + } + + [Fact] + public async Task ReturnsDataWhenInitialized() + { + await _target.EnsureInitializedAsync(); + + var value = _target.Get(); + + Assert.True(_target.Initialized); + Assert.NotNull(value); + Assert.Same(_downloadData.Metadata, value.Metadata.Downloads); + Assert.Same(_verifiedPackages.Metadata, value.Metadata.VerifiedPackages); + } + } + + public abstract class BaseFacts + { + protected readonly Mock _downloadDataClient; + protected readonly Mock _verifiedPackagesDataClient; + protected readonly Mock _popularityTransferDataClient; + protected readonly Mock _telemetryService; + protected readonly RecordingLogger _logger; + protected readonly CancellationToken _token; + protected readonly AuxiliaryFileResult _downloadData; + protected readonly AuxiliaryFileResult> _verifiedPackages; + protected readonly AuxiliaryFileResult _popularityTransfers; + protected readonly AuxiliaryDataCache _target; + + public BaseFacts(ITestOutputHelper output) + { + _downloadDataClient = new Mock(); + _verifiedPackagesDataClient = new Mock(); + _popularityTransferDataClient = new Mock(); + _telemetryService = new Mock(); + _logger = output.GetLogger(); + + _token = CancellationToken.None; + _downloadData = Data.GetAuxiliaryFileResult(new DownloadData(), "downloads-etag"); + _verifiedPackages = Data.GetAuxiliaryFileResult(new HashSet(StringComparer.OrdinalIgnoreCase), "verified-packages-etag"); + _popularityTransfers = Data.GetAuxiliaryFileResult(new PopularityTransferData(), "popularity-transfer-etag"); + + _downloadDataClient + .Setup(x => x.ReadLatestIndexedAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => _downloadData); + _verifiedPackagesDataClient + .Setup(x => x.ReadLatestAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => _verifiedPackages); + _popularityTransferDataClient + .Setup(x => x.ReadLatestIndexedAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => _popularityTransfers); + + _target = new AuxiliaryDataCache( + _downloadDataClient.Object, + _verifiedPackagesDataClient.Object, + _popularityTransferDataClient.Object, + _telemetryService.Object, + _logger); + } + + public void VerifyReadWithNoETag() + { + VerifyReadWithETags(null, null); + } + + public void VerifyReadWithETag() + { + VerifyReadWithETags(_downloadData.Metadata.ETag, _verifiedPackages.Metadata.ETag); + } + + private void VerifyReadWithETags(string downloadETag, string verifiedPackagesETag) + { + _downloadDataClient.Verify( + x => x.ReadLatestIndexedAsync( + It.Is(a => a.IfMatchETag == null && a.IfNoneMatchETag == downloadETag), + It.IsAny()), + Times.Once); + _downloadDataClient.Verify( + x => x.ReadLatestIndexedAsync( + It.IsAny(), + It.IsAny()), Times.Once); + _verifiedPackagesDataClient.Verify( + x => x.ReadLatestAsync( + It.Is(a => a.IfMatchETag == null && a.IfNoneMatchETag == verifiedPackagesETag), + It.IsAny()), + Times.Once); + _verifiedPackagesDataClient.Verify( + x => x.ReadLatestAsync( + It.IsAny(), + It.IsAny()), + Times.Once); + } + + public void VerifyNoRead() + { + _downloadDataClient.Verify( + x => x.ReadLatestIndexedAsync( + It.IsAny(), + It.IsAny()), Times.Never); + _verifiedPackagesDataClient.Verify( + x => x.ReadLatestAsync( + It.IsAny(), + It.IsAny()), Times.Never); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchService/AuxiliaryDataFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchService/AuxiliaryDataFacts.cs new file mode 100644 index 000000000..1eec5a40f --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchService/AuxiliaryDataFacts.cs @@ -0,0 +1,125 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.AzureSearch.Support; +using Xunit; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class AuxiliaryDataFacts + { + public class IsVerified : BaseFacts + { + [Fact] + public void VerifiedWhenInSet() + { + _target.VerifiedPackages.Data.Add("NuGet.Versioning"); + + var actual = _target.IsVerified("nuget.versioning"); + + Assert.True(actual); + } + + [Fact] + public void NotVerifiedWhenNotInSet() + { + var actual = _target.IsVerified("nuget.versioning"); + + Assert.False(actual); + } + } + + public class GetTotalDownloadCount : BaseFacts + { + [Fact] + public void ZeroWhenUnknownId() + { + var actual = _target.GetTotalDownloadCount("nuget.versioning"); + + Assert.Equal(0, actual); + } + + [Fact] + public void ReturnsTotal() + { + _target.Downloads.Data.SetDownloadCount("NuGet.Versioning", "1.0.0", 2); + _target.Downloads.Data.SetDownloadCount("NuGet.Versioning", "3.0.0-alpha", 23); + + var actual = _target.GetTotalDownloadCount("nuget.versioning"); + + Assert.Equal(25, actual); + } + } + + public class GetDownloadCount : BaseFacts + { + [Fact] + public void ZeroWhenUnknownId() + { + var actual = _target.GetDownloadCount("nuget.versioning", "1.0.0"); + + Assert.Equal(0, actual); + } + + [Fact] + public void ZeroWhenUnknownVersion() + { + _target.Downloads.Data.SetDownloadCount("NuGet.Versioning", "1.0.0", 2); + + var actual = _target.GetDownloadCount("nuget.versioning", "2.0.0"); + + Assert.Equal(0, actual); + } + + [Fact] + public void ReturnsCount() + { + _target.Downloads.Data.SetDownloadCount("NuGet.Versioning", "1.0.0", 2); + _target.Downloads.Data.SetDownloadCount("NuGet.Versioning", "3.0.0-alpha", 23); + + var actual = _target.GetDownloadCount("nuget.versioning", "3.0.0-ALPHA"); + + Assert.Equal(23, actual); + } + } + + public class Metadata + { + [Fact] + public void UsesSameMetadataInstances() + { + var downloadData = Data.GetAuxiliaryFileResult(new DownloadData(), string.Empty); + var verifiedPackages = Data.GetAuxiliaryFileResult(new HashSet(StringComparer.OrdinalIgnoreCase), string.Empty); + var popularityTransfers = Data.GetAuxiliaryFileResult(new PopularityTransferData(), string.Empty); + + var target = new AuxiliaryData( + DateTimeOffset.MaxValue, + downloadData, + verifiedPackages, + popularityTransfers); + + Assert.Equal(DateTimeOffset.MaxValue, target.Metadata.Loaded); + Assert.Same(downloadData.Metadata, target.Metadata.Downloads); + Assert.Same(verifiedPackages.Metadata, target.Metadata.VerifiedPackages); + Assert.Same(popularityTransfers.Metadata, target.Metadata.PopularityTransfers); + } + } + + public abstract class BaseFacts + { + protected readonly AuxiliaryData _target; + + public BaseFacts() + { + _target = new AuxiliaryData( + DateTimeOffset.MinValue, + Data.GetAuxiliaryFileResult(new DownloadData(), string.Empty), + Data.GetAuxiliaryFileResult(new HashSet(StringComparer.OrdinalIgnoreCase), string.Empty), + Data.GetAuxiliaryFileResult(new PopularityTransferData(), string.Empty)); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchService/AuxiliaryFileReloaderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchService/AuxiliaryFileReloaderFacts.cs new file mode 100644 index 000000000..83078960b --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchService/AuxiliaryFileReloaderFacts.cs @@ -0,0 +1,138 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Services.AzureSearch.Wrappers; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class AuxiliaryFileReloaderFacts + { + public class ReloadContinuouslyAsync : BaseFacts + { + public ReloadContinuouslyAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task CanBeCancelledImmediately() + { + _cts.Cancel(); + + await _target.ReloadContinuouslyAsync(_cts.Token); + + _cache.Verify(x => x.TryLoadAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task ReloadsUntilCancelled() + { + CancelAfter(reloads: 5); + _config.AuxiliaryDataReloadFrequency = TimeSpan.Zero; + + await _target.ReloadContinuouslyAsync(_cts.Token); + + _cache.Verify(x => x.TryLoadAsync(_cts.Token), Times.Exactly(5)); + _cache.Verify(x => x.TryLoadAsync(It.IsAny()), Times.Exactly(5)); + } + + [Fact] + public async Task UsesReloadFrequencyOnSuccess() + { + CancelAfter(reloads: 2); + _config.AuxiliaryDataReloadFrequency = TimeSpan.FromMilliseconds(100); + _config.AuxiliaryDataReloadFailureRetryFrequency = TimeSpan.Zero; + + await _target.ReloadContinuouslyAsync(_cts.Token); + + _cache.Verify(x => x.TryLoadAsync(It.IsAny()), Times.Exactly(2)); + _systemTime.Verify(x => x.Delay(_config.AuxiliaryDataReloadFrequency, _cts.Token), Times.Once); + _systemTime.Verify(x => x.Delay(It.IsAny(), It.IsAny()), Times.Once); + } + + [Fact] + public async Task UsesReloadFailureRetryFrequencyOnSuccess() + { + int count = 0; + _cache + .Setup(x => x.TryLoadAsync(It.IsAny())) + .Returns(() => + { + count++; + if (count >= 2) + { + _cts.Cancel(); + } + + throw new InvalidOperationException("Please retry later."); + }); + _config.AuxiliaryDataReloadFrequency = TimeSpan.Zero; + _config.AuxiliaryDataReloadFailureRetryFrequency = TimeSpan.FromMilliseconds(100); + + await _target.ReloadContinuouslyAsync(_cts.Token); + + _cache.Verify(x => x.TryLoadAsync(It.IsAny()), Times.Exactly(2)); + _systemTime.Verify(x => x.Delay(_config.AuxiliaryDataReloadFailureRetryFrequency, _cts.Token), Times.Once); + _systemTime.Verify(x => x.Delay(It.IsAny(), It.IsAny()), Times.Once); + } + } + + public abstract class BaseFacts + { + protected readonly Mock _cache; + protected readonly Mock _systemTime; + protected readonly SearchServiceConfiguration _config; + protected readonly Mock> _options; + protected readonly RecordingLogger _logger; + protected readonly CancellationTokenSource _cts; + protected readonly AuxiliaryFileReloader _target; + + public BaseFacts(ITestOutputHelper output) + { + _cache = new Mock(); + _systemTime = new Mock(); + _config = new SearchServiceConfiguration(); + _options = new Mock>(); + _logger = output.GetLogger(); + + _cts = new CancellationTokenSource(); + + // Default test behavior is to cancel after the first invocation otherwise it is very easy to loop + // forever, which is annoying for the person writing the tests. + CancelAfter(reloads: 1); + + _config.AuxiliaryDataReloadFrequency = TimeSpan.FromMilliseconds(100); + _config.AuxiliaryDataReloadFailureRetryFrequency = TimeSpan.FromMilliseconds(20); + _options.Setup(x => x.Value).Returns(() => _config); + + _target = new AuxiliaryFileReloader( + _cache.Object, + _systemTime.Object, + _options.Object, + _logger); + } + + protected void CancelAfter(int reloads) + { + int count = 0; + _cache + .Setup(x => x.TryLoadAsync(It.IsAny())) + .Returns(Task.CompletedTask) + .Callback(() => + { + count++; + if (count >= reloads) + { + _cts.Cancel(); + } + }); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchService/AzureSearchServiceFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchService/AzureSearchServiceFacts.cs new file mode 100644 index 000000000..f355a8979 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchService/AzureSearchServiceFacts.cs @@ -0,0 +1,463 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using Moq; +using NuGet.Services.AzureSearch.Wrappers; +using NuGetGallery; +using Xunit; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class AzureSearchServiceFacts + { + public class V2SearchAsync : BaseFacts + { + [Fact] + public async Task SearchIndexAndEmptyOperation() + { + _v2Request.IgnoreFilter = false; + _operation = IndexOperation.Empty(); + + var response = await _target.V2SearchAsync(_v2Request); + + Assert.Same(_v2Response, response); + _operationBuilder.Verify( + x => x.V2SearchWithSearchIndex(_v2Request), + Times.Once); + _searchOperations.Verify( + x => x.SearchAsync(It.IsAny(), It.IsAny()), + Times.Never); + _responseBuilder.Verify( + x => x.EmptyV2(_v2Request), + Times.Once); + } + + [Fact] + public async Task SearchIndexAndSearchOperation() + { + _v2Request.IgnoreFilter = false; + + var response = await _target.V2SearchAsync(_v2Request); + + Assert.Same(_v2Response, response); + _operationBuilder.Verify( + x => x.V2SearchWithSearchIndex(_v2Request), + Times.Once); + _searchOperations.Verify( + x => x.SearchAsync(_text, _parameters), + Times.Once); + _responseBuilder.Verify( + x => x.V2FromSearch(_v2Request, _text, _parameters, _searchResult, It.Is(t => t > TimeSpan.Zero)), + Times.Once); + _telemetryService.Verify( + x => x.TrackV2SearchQueryWithSearchIndex(It.Is(t => t > TimeSpan.Zero)), + Times.Once); + } + + [Fact] + public async Task ShouldThrowOnInvalidAdvancedSearchOnHijackIndex() + { + _v2Request.IgnoreFilter = true; + _v2Request.SortBy = V2SortBy.TotalDownloadsAsc; + await Assert.ThrowsAsync(async () => + { + await _target.V2SearchAsync(_v2Request); + }); + + _v2Request.SortBy = V2SortBy.TotalDownloadsDesc; + await Assert.ThrowsAsync(async () => + { + await _target.V2SearchAsync(_v2Request); + }); + + _v2Request.SortBy = V2SortBy.Popularity; // This is valid + _v2Request.PackageType = "dotnettool"; // This should make the request crash + await Assert.ThrowsAsync(async () => + { + await _target.V2SearchAsync(_v2Request); + }); + } + + [Fact] + public async Task SearchIndexAndGetOperation() + { + _v2Request.IgnoreFilter = false; + _operation = IndexOperation.Get(_key); + + var response = await _target.V2SearchAsync(_v2Request); + + Assert.Same(_v2Response, response); + _operationBuilder.Verify( + x => x.V2SearchWithSearchIndex(_v2Request), + Times.Once); + _searchOperations.Verify( + x => x.GetOrNullAsync(_key), + Times.Once); + _responseBuilder.Verify( + x => x.V2FromSearchDocument(_v2Request, _key, _searchDocument, It.Is(t => t > TimeSpan.Zero)), + Times.Once); + _telemetryService.Verify( + x => x.TrackV2GetDocumentWithSearchIndex(It.Is(t => t > TimeSpan.Zero)), + Times.Once); + } + + [Fact] + public async Task HijackIndexAndEmptyOperation() + { + _v2Request.IgnoreFilter = true; + _operation = IndexOperation.Empty(); + + var response = await _target.V2SearchAsync(_v2Request); + + Assert.Same(_v2Response, response); + _operationBuilder.Verify( + x => x.V2SearchWithHijackIndex(_v2Request), + Times.Once); + _searchOperations.Verify( + x => x.SearchAsync(It.IsAny(), It.IsAny()), + Times.Never); + _responseBuilder.Verify( + x => x.EmptyV2(_v2Request), + Times.Once); + } + + [Fact] + public async Task HijackIndexAndSearchOperation() + { + _v2Request.IgnoreFilter = true; + + var response = await _target.V2SearchAsync(_v2Request); + + Assert.Same(_v2Response, response); + _operationBuilder.Verify( + x => x.V2SearchWithHijackIndex(_v2Request), + Times.Once); + _hijackOperations.Verify( + x => x.SearchAsync(_text, _parameters), + Times.Once); + _responseBuilder.Verify( + x => x.V2FromHijack(_v2Request, _text, _parameters, _hijackResult, It.Is(t => t > TimeSpan.Zero)), + Times.Once); + _telemetryService.Verify( + x => x.TrackV2SearchQueryWithHijackIndex(It.Is(t => t > TimeSpan.Zero)), + Times.Once); + } + + [Theory] + [InlineData(false, false, false)] + [InlineData(true, false, false)] + [InlineData(false, true, true)] + [InlineData(true, true, true)] + public async Task HijackIndexAndGetOperation(bool includePrerelease, bool includeSemVer2, bool returned) + { + _v2Request.IgnoreFilter = true; + _v2Request.IncludePrerelease = includePrerelease; + _v2Request.IncludeSemVer2 = includeSemVer2; + _hijackDocument.Prerelease = true; + _hijackDocument.SemVerLevel = SemVerLevelKey.SemVer2; + _operation = IndexOperation.Get(_key); + var expectedDocument = returned ? _hijackDocument : null; + + var response = await _target.V2SearchAsync(_v2Request); + + Assert.Same(_v2Response, response); + _operationBuilder.Verify( + x => x.V2SearchWithHijackIndex(_v2Request), + Times.Once); + _hijackOperations.Verify( + x => x.GetOrNullAsync(_key), + Times.Once); + _responseBuilder.Verify( + x => x.V2FromHijackDocument(_v2Request, _key, expectedDocument, It.Is(t => t > TimeSpan.Zero)), + Times.Once); + _telemetryService.Verify( + x => x.TrackV2GetDocumentWithHijackIndex(It.Is(t => t > TimeSpan.Zero)), + Times.Once); + } + } + + public class V3SearchAsync : BaseFacts + { + [Fact] + public async Task SearchIndexAndEmptyOperation() + { + _operation = IndexOperation.Empty(); + + var response = await _target.V3SearchAsync(_v3Request); + + Assert.Same(_v3Response, response); + _operationBuilder.Verify( + x => x.V3Search(_v3Request), + Times.Once); + _searchOperations.Verify( + x => x.SearchAsync(It.IsAny(), It.IsAny()), + Times.Never); + _responseBuilder.Verify( + x => x.EmptyV3(_v3Request), + Times.Once); + } + + [Fact] + public async Task SearchIndexAndSearchOperation() + { + var response = await _target.V3SearchAsync(_v3Request); + + Assert.Same(_v3Response, response); + _operationBuilder.Verify( + x => x.V3Search(_v3Request), + Times.Once); + _searchOperations.Verify( + x => x.SearchAsync(_text, _parameters), + Times.Once); + _responseBuilder.Verify( + x => x.V3FromSearch(_v3Request, _text, _parameters, _searchResult, It.Is(t => t > TimeSpan.Zero)), + Times.Once); + _telemetryService.Verify( + x => x.TrackV3SearchQuery(It.Is(t => t > TimeSpan.Zero)), + Times.Once); + } + + [Fact] + public async Task SearchIndexAndGetOperation() + { + _operation = IndexOperation.Get(_key); + + var response = await _target.V3SearchAsync(_v3Request); + + Assert.Same(_v3Response, response); + _operationBuilder.Verify( + x => x.V3Search(_v3Request), + Times.Once); + _searchOperations.Verify( + x => x.GetOrNullAsync(_key), + Times.Once); + _responseBuilder.Verify( + x => x.V3FromSearchDocument(_v3Request, _key, _searchDocument, It.Is(t => t > TimeSpan.Zero)), + Times.Once); + _telemetryService.Verify( + x => x.TrackV3GetDocument(It.Is(t => t > TimeSpan.Zero)), + Times.Once); + } + } + + public class AutocompleteAsync : BaseFacts + { + [Fact] + public async Task SearchIndexAndEmptyOperation() + { + _operation = IndexOperation.Empty(); + + var response = await _target.AutocompleteAsync(_autocompleteRequest); + + Assert.Same(_autocompleteResponse, response); + _operationBuilder.Verify( + x => x.Autocomplete(_autocompleteRequest), + Times.Once); + _searchOperations.Verify( + x => x.SearchAsync(It.IsAny(), It.IsAny()), + Times.Never); + _responseBuilder.Verify( + x => x.EmptyAutocomplete(_autocompleteRequest), + Times.Once); + } + + [Fact] + public async Task SearchIndexAndSearchOperation() + { + var response = await _target.AutocompleteAsync(_autocompleteRequest); + + Assert.Same(_autocompleteResponse, response); + _operationBuilder.Verify( + x => x.Autocomplete(_autocompleteRequest), + Times.Once); + _searchOperations.Verify( + x => x.SearchAsync(_text, _parameters), + Times.Once); + _responseBuilder.Verify( + x => x.AutocompleteFromSearch(_autocompleteRequest, _text, _parameters, _searchResult, It.Is(t => t > TimeSpan.Zero)), + Times.Once); + _telemetryService.Verify( + x => x.TrackAutocompleteQuery(It.Is(t => t > TimeSpan.Zero)), + Times.Once); + } + + [Fact] + public async Task SearchIndexAndGetOperation() + { + _operation = IndexOperation.Get(_key); + + var ex = await Assert.ThrowsAsync(() => _target.AutocompleteAsync(_autocompleteRequest)); + Assert.Equal("The operation type Get is not supported.", ex.Message); + _operationBuilder.Verify( + x => x.Autocomplete(_autocompleteRequest), + Times.Once); + _searchOperations.Verify( + x => x.GetOrNullAsync(It.IsAny()), + Times.Never); + _telemetryService.Verify( + x => x.TrackV3GetDocument(It.IsAny()), + Times.Never); + } + } + + public abstract class BaseFacts + { + protected readonly Mock _operationBuilder; + protected readonly Mock _searchIndex; + protected readonly Mock _searchOperations; + protected readonly Mock _hijackIndex; + protected readonly Mock _hijackOperations; + protected readonly Mock _responseBuilder; + protected readonly Mock _telemetryService; + protected readonly V2SearchRequest _v2Request; + protected readonly V3SearchRequest _v3Request; + protected readonly AutocompleteRequest _autocompleteRequest; + protected readonly string _key; + protected readonly string _text; + protected readonly SearchParameters _parameters; + protected IndexOperation _operation; + protected readonly DocumentSearchResult _searchResult; + protected readonly SearchDocument.Full _searchDocument; + protected readonly DocumentSearchResult _hijackResult; + protected readonly HijackDocument.Full _hijackDocument; + protected readonly V2SearchResponse _v2Response; + protected readonly V3SearchResponse _v3Response; + protected readonly AutocompleteResponse _autocompleteResponse; + protected readonly AzureSearchService _target; + + public BaseFacts() + { + _operationBuilder = new Mock(); + _searchIndex = new Mock(); + _searchOperations = new Mock(); + _hijackIndex = new Mock(); + _hijackOperations = new Mock(); + _responseBuilder = new Mock(); + _telemetryService = new Mock(); + + _v2Request = new V2SearchRequest(); + _v3Request = new V3SearchRequest(); + _autocompleteRequest = new AutocompleteRequest(); + _key = "key"; + _text = "search"; + _parameters = new SearchParameters(); + _operation = IndexOperation.Search(_text, _parameters); + _searchResult = new DocumentSearchResult(); + _searchDocument = new SearchDocument.Full(); + _hijackResult = new DocumentSearchResult(); + _hijackDocument = new HijackDocument.Full(); + _v2Response = new V2SearchResponse(); + _v3Response = new V3SearchResponse(); + _autocompleteResponse = new AutocompleteResponse(); + + _operationBuilder + .Setup(x => x.V2SearchWithHijackIndex(It.IsAny())) + .Returns(() => _operation); + _operationBuilder + .Setup(x => x.V2SearchWithSearchIndex(It.IsAny())) + .Returns(() => _operation); + _operationBuilder + .Setup(x => x.V3Search(It.IsAny())) + .Returns(() => _operation); + _operationBuilder + .Setup(x => x.Autocomplete(It.IsAny())) + .Returns(() => _operation); + + // Set up the search to take at least one millisecond. This allows us to test the measured duration. + _searchOperations + .Setup(x => x.SearchAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => _searchResult) + .Callback(() => Thread.Sleep(TimeSpan.FromMilliseconds(1))); + _searchOperations + .Setup(x => x.GetOrNullAsync(It.IsAny())) + .ReturnsAsync(() => _searchDocument) + .Callback(() => Thread.Sleep(TimeSpan.FromMilliseconds(1))); + _hijackOperations + .Setup(x => x.SearchAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => _hijackResult) + .Callback(() => Thread.Sleep(TimeSpan.FromMilliseconds(1))); + _hijackOperations + .Setup(x => x.GetOrNullAsync(It.IsAny())) + .ReturnsAsync(() => _hijackDocument) + .Callback(() => Thread.Sleep(TimeSpan.FromMilliseconds(1))); + + _responseBuilder + .Setup(x => x.V2FromHijack( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny())) + .Returns(() => _v2Response); + _responseBuilder + .Setup(x => x.V2FromHijackDocument( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(() => _v2Response); + _responseBuilder + .Setup(x => x.V2FromSearch( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny())) + .Returns(() => _v2Response); + _responseBuilder + .Setup(x => x.V2FromSearchDocument( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(() => _v2Response); + _responseBuilder + .Setup(x => x.V3FromSearch( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny())) + .Returns(() => _v3Response); + _responseBuilder + .Setup(x => x.V3FromSearchDocument( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(() => _v3Response); + _responseBuilder + .Setup(x => x.AutocompleteFromSearch( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny())) + .Returns(() => _autocompleteResponse); + _responseBuilder + .Setup(x => x.EmptyV2(It.IsAny())) + .Returns(() => _v2Response); + _responseBuilder + .Setup(x => x.EmptyV3(It.IsAny())) + .Returns(() => _v3Response); + _responseBuilder + .Setup(x => x.EmptyAutocomplete(It.IsAny())) + .Returns(() => _autocompleteResponse); + + _searchIndex.Setup(x => x.Documents).Returns(() => _searchOperations.Object); + _hijackIndex.Setup(x => x.Documents).Returns(() => _hijackOperations.Object); + + _target = new AzureSearchService( + _operationBuilder.Object, + _searchIndex.Object, + _hijackIndex.Object, + _responseBuilder.Object, + _telemetryService.Object); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchService/IndexOperationBuilderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchService/IndexOperationBuilderFacts.cs new file mode 100644 index 000000000..b7b8b176c --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchService/IndexOperationBuilderFacts.cs @@ -0,0 +1,572 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Azure.Search.Models; +using Moq; +using Xunit; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class IndexOperationBuilderFacts + { + public class Autocomplete : Facts + { + public override IndexOperation Build() + { + return Target.Autocomplete(AutocompleteRequest); + } + + [Fact] + public void BuildsSearchOperation() + { + var actual = Target.Autocomplete(AutocompleteRequest); + + Assert.Equal(IndexOperationType.Search, actual.Type); + Assert.Same(Text, actual.SearchText); + Assert.Same(Parameters, actual.SearchParameters); + TextBuilder.Verify(x => x.Autocomplete(AutocompleteRequest), Times.Once); + ParametersBuilder.Verify(x => x.Autocomplete(AutocompleteRequest, It.IsAny()), Times.Once); + } + + [Fact] + public void ReturnsEmptyQueryForInvalidPackageType() + { + AutocompleteRequest.PackageType = "invalid package type"; + + var actual = Build(); + + Assert.Equal(IndexOperationType.Empty, actual.Type); + } + } + + public class V3Search : SearchIndexFacts + { + public override IndexOperation Build() + { + return Target.V3Search(V3SearchRequest); + } + + [Fact] + public void CallsDependenciesForGetOperation() + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + + Build(); + + TextBuilder.Verify(x => x.ParseV3Search(V3SearchRequest), Times.Once); + TextBuilder.Verify(x => x.Build(It.IsAny()), Times.Never); + ParametersBuilder.Verify(x => x.V3Search(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public void CallsDependenciesForSearchOperation() + { + Build(); + + TextBuilder.Verify(x => x.ParseV3Search(V3SearchRequest), Times.Once); + TextBuilder.Verify(x => x.Build(ParsedQuery), Times.Once); + ParametersBuilder.Verify(x => x.V3Search(V3SearchRequest, It.IsAny()), Times.Once); + } + + [Fact] + public void ReturnsEmptyQueryForInvalidPackageType() + { + V3SearchRequest.PackageType = "invalid package type"; + + var actual = Build(); + + Assert.Equal(IndexOperationType.Empty, actual.Type); + } + + [Fact] + public void BuildsSearchOperationForSingleValidPackageIdAndPackageType() + { + V3SearchRequest.PackageType = "Dependency"; + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + } + + public class V2SearchWithSearchIndex : SearchIndexFacts + { + public override IndexOperation Build() + { + return Target.V2SearchWithSearchIndex(V2SearchRequest); + } + + [Fact] + public void CallsDependenciesForGetOperation() + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + + Build(); + + TextBuilder.Verify(x => x.ParseV2Search(V2SearchRequest), Times.Once); + TextBuilder.Verify(x => x.Build(It.IsAny()), Times.Never); + ParametersBuilder.Verify(x => x.V2Search(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public void CallsDependenciesForSearchOperation() + { + Build(); + + TextBuilder.Verify(x => x.ParseV2Search(V2SearchRequest), Times.Once); + TextBuilder.Verify(x => x.Build(ParsedQuery), Times.Once); + ParametersBuilder.Verify(x => x.V2Search(V2SearchRequest, It.IsAny()), Times.Once); + } + + [Fact] + public void ReturnsEmptyQueryForInvalidPackageType() + { + V2SearchRequest.PackageType = "invalid package type"; + + var actual = Build(); + + Assert.Equal(IndexOperationType.Empty, actual.Type); + } + + [Fact] + public void BuildsSearchOperationForSingleValidPackageIdAndPackageType() + { + V2SearchRequest.PackageType = "Dependency"; + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + } + + public class V2SearchWithHijackIndex : Facts + { + public override IndexOperation Build() + { + return Target.V2SearchWithHijackIndex(V2SearchRequest); + } + + [Theory] + [MemberData(nameof(ValidIdData))] + public void BuildsGetOperationForSingleValidPackageIdAndSingleValidVersion(string id) + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { id }); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(new[] { Version }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Get, actual.Type); + Assert.Equal( + DocumentUtilities.GetHijackDocumentKey(id, Version), + actual.DocumentKey); + } + + [Theory] + [MemberData(nameof(InvalidIdData))] + public void DoesNotBuildGetOperationForSingleInvalidPackageIdAndSingleValidVersion(string id) + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { id }); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(new[] { Version }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Theory] + [InlineData("\"1.0.0\"")] + [InlineData("1.0.0.0.0")] + [InlineData("1.0.0.a")] + [InlineData("1.0.0.-alpha")] + [InlineData("1.0.0-beta.01")] + [InlineData("alpha")] + [InlineData("")] + public void DoesNotBuildGetOperationForSingleValidPackageIdAndSingleInvalidVersion(string version) + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(new[] { version }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Theory] + [InlineData("1.0.0", "1.0.0")] + [InlineData("1.0.0-BETA", "1.0.0-BETA")] + [InlineData("1.0.0-beta01", "1.0.0-beta01")] + [InlineData("1.0.0-beta.2", "1.0.0-beta.2")] + [InlineData("1.0.0.0", "1.0.0")] + [InlineData("1.0.0-ALPHA+git", "1.0.0-ALPHA")] + [InlineData("1.0.0-alpha+git", "1.0.0-alpha")] + [InlineData("1.0.00-alpha", "1.0.0-alpha")] + [InlineData("1.0.01-alpha", "1.0.1-alpha")] + [InlineData(" 1.0.0 ", "1.0.0")] + public void NormalizesVersion(string version, string normalized) + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(new[] { version }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Get, actual.Type); + Assert.Equal( + DocumentUtilities.GetHijackDocumentKey(Id, normalized), + actual.DocumentKey); + } + + [Fact] + public void IgnoresFiltersWithSpecificPackageIdAndVersion() + { + V2SearchRequest.IncludePrerelease = false; + V2SearchRequest.IncludeSemVer2 = false; + var prereleaseSemVer2 = "1.0.0-beta.1"; + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(new[] { prereleaseSemVer2 }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Get, actual.Type); + Assert.Equal( + DocumentUtilities.GetHijackDocumentKey(Id, prereleaseSemVer2), + actual.DocumentKey); + } + + [Fact] + public void DoesNotBuildGetOperationForNonPackageIdAndVersion() + { + ParsedQuery.Grouping[QueryField.Id] = new HashSet(new[] { Id }); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(new[] { Version }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void DoesNotBuildGetOperationForMultiplePackageIds() + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id, "A" }); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(new[] { Version }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void DoesNotBuildGetOperationForMultipleVersions() + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(new[] { Version, "9.9.9" }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void DoesNotBuildGetOperationForPackageIdVersionAndExtra() + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(new[] { Version }); + ParsedQuery.Grouping[QueryField.Description] = new HashSet(new[] { "hi" }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void DoesNotBuildGetOperationForEmptyPackageId() + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(new[] { Version }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void DoesNotBuildGetOperationForEmptyVersion() + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void DoesNotBuildGetOperationForSkippingFirstItem() + { + V2SearchRequest.Skip = 1; + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(new[] { Version }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void DoesNotBuildGetOperationForNoTake() + { + V2SearchRequest.Take = 0; + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(new[] { Version }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void CallsDependenciesForGetOperation() + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(new[] { Version }); + + Build(); + + TextBuilder.Verify(x => x.ParseV2Search(V2SearchRequest), Times.Once); + TextBuilder.Verify(x => x.Build(It.IsAny()), Times.Never); + ParametersBuilder.Verify(x => x.V2Search(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public void CallsDependenciesForSearchOperation() + { + Build(); + + TextBuilder.Verify(x => x.ParseV2Search(V2SearchRequest), Times.Once); + TextBuilder.Verify(x => x.Build(ParsedQuery), Times.Once); + ParametersBuilder.Verify(x => x.V2Search(V2SearchRequest, It.IsAny()), Times.Once); + } + } + + public abstract class SearchIndexFacts : Facts + { + [Theory] + [MemberData(nameof(ValidIdData))] + public void BuildsGetOperationForSingleValidPackageId(string id) + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { id }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Get, actual.Type); + Assert.Equal( + DocumentUtilities.GetSearchDocumentKey(id, SearchFilters.Default), + actual.DocumentKey); + } + + [Theory] + [MemberData(nameof(InvalidIdData))] + public void DoesNotBuildGetOperationForInvalidPackageId(string id) + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { id }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void DoesNotBuildGetOperationForEmptyPackageIdQuery() + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void DoesNotBuildGetOperationForIdQuery() + { + ParsedQuery.Grouping[QueryField.Id] = new HashSet(new[] { Id }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void DoesNotBuildGetOperationForSkippingFirstItem() + { + V2SearchRequest.Skip = 1; + V3SearchRequest.Skip = 1; + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void DoesNotBuildGetOperationForNoTake() + { + V2SearchRequest.Take = 0; + V3SearchRequest.Take = 0; + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void DoesNotBuildGetOperationForPackageIdVersion() + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + ParsedQuery.Grouping[QueryField.Version] = new HashSet(new[] { Version }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void DoesNotBuildGetOperationForMultipleFields() + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id }); + ParsedQuery.Grouping[QueryField.Description] = new HashSet(new[] { "hi" }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void DoesNotBuildGetOperationForMultiplePackageIds() + { + ParsedQuery.Grouping[QueryField.PackageId] = new HashSet(new[] { Id, "A" }); + + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + + [Fact] + public void BuildsSearchOperationForNonPackageIdQueries() + { + var actual = Build(); + + Assert.Equal(IndexOperationType.Search, actual.Type); + } + } + + public abstract class Facts + { + public const string Id = "NuGet.Versioning"; + public const string Version = "5.1.0"; + + public const int MaxIdLength = 100; + + public static IReadOnlyList ValidIds => new[] + { + Id, + new string('a', MaxIdLength), + "A", + "Foo__Bar", + }; + + public static IReadOnlyList InvalidIds => new[] + { + string.Empty, + " ", + "\nNuGet.Versioning", + "A,B", + "foo--bar", + "foo..bar", + " NuGet.Versioning ", + " NuGet.Versioning", + "NuGet.Versioning ", + "\"NuGet.Versioning\"", + new string('a', MaxIdLength + 1), + }; + + public static IEnumerable ValidIdData => ValidIds.Select(x => new object[] { x }); + public static IEnumerable InvalidIdData => InvalidIds.Select(x => new object[] { x }); + + public static IEnumerable TooLargeSkip => new[] + { + new object[] { 99999, IndexOperationType.Search }, + new object[] { 100000, IndexOperationType.Search }, + new object[] { 100001, IndexOperationType.Empty }, + new object[] { 100002, IndexOperationType.Empty }, + }; + + public Facts() + { + TextBuilder = new Mock(); + ParametersBuilder = new Mock(); + + AutocompleteRequest = new AutocompleteRequest { Skip = 0, Take = 20 }; + V2SearchRequest = new V2SearchRequest { Skip = 0, Take = 20 }; + V3SearchRequest = new V3SearchRequest { Skip = 0, Take = 20 }; + Text = ""; + Parameters = new SearchParameters(); + ParsedQuery = new ParsedQuery(new Dictionary>()); + + TextBuilder + .Setup(x => x.Autocomplete(It.IsAny())) + .Returns(() => Text); + TextBuilder + .Setup(x => x.ParseV2Search(It.IsAny())) + .Returns(() => ParsedQuery); + TextBuilder + .Setup(x => x.ParseV3Search(It.IsAny())) + .Returns(() => ParsedQuery); + TextBuilder + .Setup(x => x.Build(It.IsAny())) + .Returns(() => Text); + ParametersBuilder + .Setup(x => x.Autocomplete(It.IsAny(), It.IsAny())) + .Returns(() => Parameters); + ParametersBuilder + .Setup(x => x.V2Search(It.IsAny(), It.IsAny())) + .Returns(() => Parameters); + ParametersBuilder + .Setup(x => x.V3Search(It.IsAny(), It.IsAny())) + .Returns(() => Parameters); + + Target = new IndexOperationBuilder( + TextBuilder.Object, + ParametersBuilder.Object); + } + + public Mock TextBuilder { get; } + public Mock ParametersBuilder { get; } + public AutocompleteRequest AutocompleteRequest { get; } + public V2SearchRequest V2SearchRequest { get; } + public V3SearchRequest V3SearchRequest { get; } + public string Text { get; } + public SearchParameters Parameters { get; } + public ParsedQuery ParsedQuery { get; } + public IndexOperationBuilder Target { get; } + + public abstract IndexOperation Build(); + + [Theory] + [MemberData(nameof(TooLargeSkip))] + public void ReturnsEmptyOperationForTooLargeSkip(int skip, IndexOperationType expected) + { + AutocompleteRequest.Skip = skip; + V2SearchRequest.Skip = skip; + V3SearchRequest.Skip = skip; + + var actual = Build(); + + Assert.Equal(expected, actual.Type); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchParametersBuilderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchParametersBuilderFacts.cs new file mode 100644 index 000000000..47474e8bd --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchParametersBuilderFacts.cs @@ -0,0 +1,578 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Azure.Search; +using Microsoft.Azure.Search.Models; +using Moq; +using Xunit; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class SearchParametersBuilderFacts + { + public class GetSearchFilters : BaseFacts + { + [Theory] + [MemberData(nameof(AllSearchFilters))] + public void SearchFilters(bool includePrerelease, bool includeSemVer2, SearchFilters filter) + { + var request = new SearchRequest + { + IncludePrerelease = includePrerelease, + IncludeSemVer2 = includeSemVer2, + }; + + var actual = _target.GetSearchFilters(request); + + Assert.Equal(filter, actual); + } + } + + public class LastCommitTimestamp : BaseFacts + { + [Fact] + public void Defaults() + { + var output = _target.LastCommitTimestamp(); + + Assert.Equal(QueryType.Full, output.QueryType); + Assert.False(output.IncludeTotalResultCount); + Assert.Equal(new[] { "lastCommitTimestamp desc" }, output.OrderBy.ToArray()); + Assert.Equal(0, output.Skip); + Assert.Equal(1, output.Top); + Assert.Null(output.Filter); + } + } + + public class V2Search : BaseFacts + { + [Fact] + public void Defaults() + { + var request = new V2SearchRequest(); + + var output = _target.V2Search(request, isDefaultSearch: true); + + Assert.Equal(QueryType.Full, output.QueryType); + Assert.True(output.IncludeTotalResultCount); + Assert.Equal(DefaultOrderBy, output.OrderBy.ToArray()); + Assert.Equal(0, output.Skip); + Assert.Equal(0, output.Top); + Assert.Equal("searchFilters eq 'Default' and (isExcludedByDefault eq false or isExcludedByDefault eq null)", output.Filter); + } + + [Theory] + [MemberData(nameof(ValidPackageTypes))] + public void PackageTypeFiltering(string packageType) + { + var request = new V2SearchRequest + { + PackageType = packageType, + }; + + var output = _target.V2Search(request, It.IsAny()); + + Assert.Equal($"searchFilters eq 'Default' and filterablePackageTypes/any(p: p eq '{packageType.ToLowerInvariant()}')", output.Filter); + } + + [Fact] + public void InvalidPackageType() + { + var request = new V2SearchRequest + { + PackageType = "something's-weird", + }; + + var output = _target.V2Search(request, It.IsAny()); + + Assert.Equal("searchFilters eq 'Default'", output.Filter); + } + + [Fact] + public void CountOnly() + { + var request = new V2SearchRequest + { + CountOnly = true, + Skip = 10, + Take = 30, + SortBy = V2SortBy.SortableTitleAsc, + }; + + var output = _target.V2Search(request, It.IsAny()); + + Assert.Equal(QueryType.Full, output.QueryType); + Assert.True(output.IncludeTotalResultCount); + Assert.Null(output.OrderBy); + Assert.Equal(0, output.Skip); + Assert.Equal(0, output.Top); + } + + [Fact] + public void Paging() + { + var request = new V2SearchRequest + { + Skip = 10, + Take = 30, + }; + + var output = _target.V2Search(request, It.IsAny()); + + Assert.Equal(10, output.Skip); + Assert.Equal(30, output.Top); + } + + [Fact] + public void NegativeSkip() + { + var request = new V2SearchRequest + { + Skip = -10, + }; + + var output = _target.V2Search(request, It.IsAny()); + + Assert.Equal(0, output.Skip); + } + + [Fact] + public void NegativeTake() + { + var request = new V2SearchRequest + { + Take = -20, + }; + + var output = _target.V2Search(request, It.IsAny()); + + Assert.Equal(20, output.Top); + } + + [Fact] + public void TooLargeTake() + { + var request = new V2SearchRequest + { + Take = 1001, + }; + + var output = _target.V2Search(request, It.IsAny()); + + Assert.Equal(20, output.Top); + } + + [Theory] + [InlineData(false, false, "semVerLevel ne 2")] + [InlineData(true, false, "semVerLevel ne 2")] + [InlineData(false, true, null)] + [InlineData(true, true, null)] + public void IgnoreFilter(bool includePrerelease, bool includeSemVer2, string filter) + { + var request = new V2SearchRequest + { + IgnoreFilter = true, + IncludePrerelease = includePrerelease, + IncludeSemVer2 = includeSemVer2, + }; + + var output = _target.V2Search(request, It.IsAny()); + + Assert.Equal(filter, output.Filter); + } + + [Theory] + [MemberData(nameof(AllV2SortBy))] + public void SortBy(V2SortBy v2SortBy) + { + var request = new V2SearchRequest + { + SortBy = v2SortBy, + }; + var expectedOrderBy = V2SortByToOrderBy[v2SortBy]; + + var output = _target.V2Search(request, It.IsAny()); + + Assert.NotNull(output.OrderBy); + Assert.Equal(expectedOrderBy, output.OrderBy.ToArray()); + } + + [Theory] + [MemberData(nameof(AllV2SortBy))] + public void AllSortByFieldsAreSortable(V2SortBy v2SortBy) + { + var metadataProperties = typeof(BaseMetadataDocument) + .GetProperties() + .Union(typeof(SearchDocument.Full).GetProperties()) // Properties can also be in a SearchDocument (e.g: TotalDownloadCount) + .GroupBy(x => x.Name, StringComparer.OrdinalIgnoreCase) + .Select(g => g.First()) + .ToDictionary(x => x.Name, x => x, StringComparer.OrdinalIgnoreCase); + + var expectedOrderBy = V2SortByToOrderBy[v2SortBy]; + + foreach (var clause in expectedOrderBy) + { + var pieces = clause.Split(new[] { ' ' }, 2); + Assert.Equal(2, pieces.Length); + Assert.Contains(pieces[1], new[] { "asc", "desc" }); + + // This is a magic property name that refers to the document's score, not a particular property. + if (pieces[0] == "search.score()") + { + continue; + } + + Assert.Contains(pieces[0], metadataProperties.Keys); + var property = metadataProperties[pieces[0]]; + var customAttributeTypes = property + .CustomAttributes + .Select(x => x.AttributeType) + .ToArray(); + Assert.Contains(typeof(IsSortableAttribute), customAttributeTypes); + } + } + + [Theory] + [MemberData(nameof(AllSearchFiltersExpressions))] + public void SearchFilters(bool includePrerelease, bool includeSemVer2, string filter) + { + var request = new V2SearchRequest + { + IncludePrerelease = includePrerelease, + IncludeSemVer2 = includeSemVer2, + Query = "js" + }; + + var output = _target.V2Search(request, It.IsAny()); + + Assert.Equal(filter, output.Filter); + } + } + + public class V3Search : BaseFacts + { + [Fact] + public void Defaults() + { + var request = new V3SearchRequest(); + + var output = _target.V3Search(request, isDefaultSearch: true); + + Assert.Equal(QueryType.Full, output.QueryType); + Assert.True(output.IncludeTotalResultCount); + Assert.Equal(DefaultOrderBy, output.OrderBy.ToArray()); + Assert.Equal(0, output.Skip); + Assert.Equal(0, output.Top); + Assert.Equal("searchFilters eq 'Default' and (isExcludedByDefault eq false or isExcludedByDefault eq null)", output.Filter); + } + + [Theory] + [MemberData(nameof(ValidPackageTypes))] + public void PackageTypeFiltering(string packageType) + { + var request = new V3SearchRequest + { + PackageType = packageType, + }; + + var output = _target.V3Search(request, It.IsAny()); + + Assert.Equal($"searchFilters eq 'Default' and filterablePackageTypes/any(p: p eq '{packageType.ToLowerInvariant()}')", output.Filter); + } + + [Fact] + public void InvalidPackageType() + { + var request = new V3SearchRequest + { + PackageType = "something's-weird", + }; + + var output = _target.V3Search(request, It.IsAny()); + + Assert.Equal("searchFilters eq 'Default'", output.Filter); + } + + [Fact] + public void Paging() + { + var request = new V3SearchRequest + { + Skip = 10, + Take = 30, + }; + + var output = _target.V3Search(request, It.IsAny()); + + Assert.Equal(10, output.Skip); + Assert.Equal(30, output.Top); + } + + [Fact] + public void NegativeSkip() + { + var request = new V3SearchRequest + { + Skip = -10, + }; + + var output = _target.V3Search(request, It.IsAny()); + + Assert.Equal(0, output.Skip); + } + + [Fact] + public void NegativeTake() + { + var request = new V3SearchRequest + { + Take = -20, + }; + + var output = _target.V3Search(request, It.IsAny()); + + Assert.Equal(20, output.Top); + } + + [Fact] + public void TooLargeTake() + { + var request = new V3SearchRequest + { + Take = 1001, + }; + + var output = _target.V3Search(request, It.IsAny()); + + Assert.Equal(20, output.Top); + } + + [Theory] + [MemberData(nameof(AllSearchFiltersExpressions))] + public void SearchFilters(bool includePrerelease, bool includeSemVer2, string filter) + { + var request = new V3SearchRequest + { + IncludePrerelease = includePrerelease, + IncludeSemVer2 = includeSemVer2, + Query = "js" + }; + + var output = _target.V3Search(request, It.IsAny()); + + Assert.Equal(filter, output.Filter); + } + } + + public class Autocomplete : BaseFacts + { + [Fact] + public void PackageIdsDefaults() + { + var request = new AutocompleteRequest(); + request.Type = AutocompleteRequestType.PackageIds; + + var output = _target.Autocomplete(request, isDefaultSearch: true); + + Assert.Equal(QueryType.Full, output.QueryType); + Assert.True(output.IncludeTotalResultCount); + Assert.Equal(DefaultOrderBy, output.OrderBy.ToArray()); + Assert.Equal(0, output.Skip); + Assert.Equal(0, output.Top); + Assert.Equal("searchFilters eq 'Default' and (isExcludedByDefault eq false or isExcludedByDefault eq null)", output.Filter); + Assert.Single(output.Select); + Assert.Equal(IndexFields.PackageId, output.Select[0]); + } + + [Fact] + public void PackageVersionsDefaults() + { + var request = new AutocompleteRequest(); + request.Type = AutocompleteRequestType.PackageVersions; + + var output = _target.Autocomplete(request, isDefaultSearch: true); + + Assert.Equal(QueryType.Full, output.QueryType); + Assert.True(output.IncludeTotalResultCount); + Assert.Equal(DefaultOrderBy, output.OrderBy.ToArray()); + Assert.Equal(0, output.Skip); + Assert.Equal(1, output.Top); + Assert.Equal("searchFilters eq 'Default' and (isExcludedByDefault eq false or isExcludedByDefault eq null)", output.Filter); + Assert.Single(output.Select); + Assert.Equal(IndexFields.Search.Versions, output.Select[0]); + } + + [Fact] + public void Paging() + { + var request = new AutocompleteRequest + { + Skip = 10, + Take = 30, + Type = AutocompleteRequestType.PackageIds, + }; + + var output = _target.Autocomplete(request, It.IsAny()); + + Assert.Equal(10, output.Skip); + Assert.Equal(30, output.Top); + } + + [Fact] + public void PackageVersionsPaging() + { + var request = new AutocompleteRequest + { + Skip = 10, + Take = 30, + Type = AutocompleteRequestType.PackageVersions, + }; + + var output = _target.Autocomplete(request, It.IsAny()); + + Assert.Equal(0, output.Skip); + Assert.Equal(1, output.Top); + } + + [Fact] + public void NegativeSkip() + { + var request = new AutocompleteRequest + { + Skip = -10, + Type = AutocompleteRequestType.PackageIds, + }; + + var output = _target.Autocomplete(request, It.IsAny()); + + Assert.Equal(0, output.Skip); + } + + [Fact] + public void NegativeTake() + { + var request = new AutocompleteRequest + { + Take = -20, + Type = AutocompleteRequestType.PackageIds, + }; + + var output = _target.Autocomplete(request, It.IsAny()); + + Assert.Equal(20, output.Top); + } + + [Fact] + public void TooLargeTake() + { + var request = new AutocompleteRequest + { + Type = AutocompleteRequestType.PackageIds, + Take = 1001, + }; + + var output = _target.Autocomplete(request, It.IsAny()); + + Assert.Equal(20, output.Top); + } + + [Theory] + [MemberData(nameof(AllSearchFiltersExpressions))] + public void SearchFilters(bool includePrerelease, bool includeSemVer2, string filter) + { + var request = new AutocompleteRequest + { + IncludePrerelease = includePrerelease, + IncludeSemVer2 = includeSemVer2, + Query = "js" + }; + + var output = _target.Autocomplete(request, It.IsAny()); + + Assert.Equal(filter, output.Filter); + } + + [Theory] + [MemberData(nameof(ValidPackageTypes))] + public void PackageTypeFiltering(string packageType) + { + var request = new AutocompleteRequest + { + PackageType = packageType, + }; + + var output = _target.Autocomplete(request, It.IsAny()); + + Assert.Equal($"searchFilters eq 'Default' and filterablePackageTypes/any(p: p eq '{packageType.ToLowerInvariant()}')", output.Filter); + } + + [Fact] + public void InvalidPackageType() + { + var request = new AutocompleteRequest + { + PackageType = "something's-weird", + }; + + var output = _target.Autocomplete(request, It.IsAny()); + + Assert.Equal("searchFilters eq 'Default'", output.Filter); + } + } + + public abstract class BaseFacts + { + protected readonly SearchParametersBuilder _target; + + public static string[] DefaultOrderBy => new[] { "search.score() desc", "created desc" }; + + public static IReadOnlyDictionary V2SortByToOrderBy => new Dictionary + { + { V2SortBy.LastEditedDesc, new[] { "lastEdited desc", "created desc" } }, + { V2SortBy.Popularity, DefaultOrderBy }, + { V2SortBy.PublishedDesc, new[] { "published desc", "created desc" } }, + { V2SortBy.SortableTitleAsc, new[] { "sortableTitle asc", "created asc" } }, + { V2SortBy.SortableTitleDesc, new[] { "sortableTitle desc", "created desc" } }, + { V2SortBy.CreatedAsc, new[] { "created asc" } }, + { V2SortBy.CreatedDesc, new[] { "created desc" } }, + { V2SortBy.TotalDownloadsAsc, new[] { "totalDownloadCount asc", "created asc"} }, + { V2SortBy.TotalDownloadsDesc, new[] { "totalDownloadCount desc", "created desc"} }, + }; + + public static IEnumerable AllSearchFilters => new[] + { + new object[] { false, false, SearchFilters.Default }, + new object[] { true, false, SearchFilters.IncludePrerelease }, + new object[] { false, true, SearchFilters.IncludeSemVer2 }, + new object[] { true, true, SearchFilters.IncludePrereleaseAndSemVer2 }, + }; + + public static IEnumerable AllSearchFiltersExpressions => AllSearchFilters + .Select(x => new[] { x[0], x[1], $"searchFilters eq '{x[2]}'" }); + + public static IEnumerable AllV2SortBy => Enum + .GetValues(typeof(V2SortBy)) + .Cast() + .Select(x => new object[] { x }); + + + public static IEnumerable ValidPackageTypes => new[] + { + new object[] { "Dependency" }, + new object[] { "DotnetTool" }, + new object[] { "Template" }, + new object[] { "PackageType.With.Dots" }, + new object[] { "PackageType-With-Hyphens" }, + new object[] { "PackageType_With_Underscores" }, + }; + + public BaseFacts() + { + _target = new SearchParametersBuilder(); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchResponseBuilderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchResponseBuilderFacts.cs new file mode 100644 index 000000000..4f6d648d6 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchResponseBuilderFacts.cs @@ -0,0 +1,1519 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using System.Net; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Options; +using Moq; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.AzureSearch.Support; +using NuGet.Versioning; +using Xunit; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class SearchResponseBuilderFacts + { + public class V2FromHijack : BaseFacts + { + [Fact] + public void ShowsUnlisted() + { + _hijackResult.Results[0].Document.Listed = false; + + var response = Target.V2FromHijack( + _v2Request, + _text, + _searchParameters, + _hijackResult, + _duration); + + Assert.False(response.Data[0].Listed); + } + + [Theory] + [InlineData(false, true, true, false, false)] + [InlineData(true, false, false, true, true)] + public void UsesCorrectSetOfLatestBooleans( + bool includeSemVer2, + bool isLatestStableSemVer1, + bool isLatestSemVer1, + bool isLatestStableSemVer2, + bool isLatestSemVer2) + { + _v2Request.IncludeSemVer2 = includeSemVer2; + var doc = _hijackResult.Results[0].Document; + doc.IsLatestStableSemVer1 = isLatestStableSemVer1; + doc.IsLatestSemVer1 = isLatestSemVer1; + doc.IsLatestStableSemVer2 = isLatestStableSemVer2; + doc.IsLatestSemVer2 = isLatestSemVer2; + + var response = Target.V2FromHijack( + _v2Request, + _text, + _searchParameters, + _hijackResult, + _duration); + + Assert.True(response.Data[0].IsLatestStable); + Assert.True(response.Data[0].IsLatest); + } + + [Theory] + [MemberData(nameof(MissingTitles))] + public void UsesIdForMissingTitle(string title) + { + _hijackResult.Results[0].Document.Title = title; + + var response = Target.V2FromHijack( + _v2Request, + _text, + _searchParameters, + _hijackResult, + _duration); + + Assert.Equal(Data.PackageId, response.Data[0].Title); + } + + [Fact] + public void CoalescesSomeNullFields() + { + var doc = _hijackResult.Results[0].Document; + doc.OriginalVersion = null; + doc.Description = null; + doc.Summary = null; + doc.Authors = null; + doc.Tags = null; + doc.RequiresLicenseAcceptance = null; + + var response = Target.V2FromHijack( + _v2Request, + _text, + _searchParameters, + _hijackResult, + _duration); + + var result = response.Data[0]; + Assert.NotNull(result.Version); + Assert.Equal(doc.NormalizedVersion, result.Version); + Assert.Equal(string.Empty, result.Description); + Assert.Equal(string.Empty, result.Summary); + Assert.Equal(string.Empty, result.Authors); + Assert.Equal(string.Empty, result.Tags); + Assert.False(result.RequiresLicenseAcceptance); + } + + [Fact] + public void UsesVersionSpecificDownloadCount() + { + _auxiliaryData.Setup(x => x.GetDownloadCount(It.IsAny(), "7.1.2-alpha")).Returns(4); + + var response = Target.V2FromHijack( + _v2Request, + _text, + _searchParameters, + _hijackResult, + _duration); + + Assert.Equal(4, response.Data[0].DownloadCount); + } + + [Fact] + public void CanIncludeDebugInformation() + { + _v2Request.ShowDebug = true; + var docResult = _hijackResult.Results[0]; + + var response = Target.V2FromHijack( + _v2Request, + _text, + _searchParameters, + _hijackResult, + _duration); + + Assert.NotNull(response.Debug); + var actualJson = JsonConvert.SerializeObject(response.Debug, _jsonSerializerSettings); + Assert.Equal(@"{ + ""SearchRequest"": { + ""IgnoreFilter"": false, + ""CountOnly"": false, + ""SortBy"": ""Popularity"", + ""LuceneQuery"": false, + ""Skip"": 0, + ""Take"": 0, + ""IncludePrerelease"": true, + ""IncludeSemVer2"": true, + ""ShowDebug"": true + }, + ""IndexName"": ""hijack-index"", + ""IndexOperationType"": ""Search"", + ""SearchParameters"": { + ""IncludeTotalResultCount"": false, + ""QueryType"": ""simple"", + ""SearchMode"": ""any"" + }, + ""SearchText"": ""azure storage sdk"", + ""DocumentSearchResult"": { + ""Count"": 1 + }, + ""QueryDuration"": ""00:00:00.2500000"", + ""AuxiliaryFilesMetadata"": { + ""Loaded"": ""2019-01-03T11:00:00+00:00"", + ""Downloads"": { + ""LastModified"": ""2019-01-01T11:00:00+00:00"", + ""LoadDuration"": ""00:00:15"", + ""FileSize"": 1234, + ""ETag"": ""\""etag-a\"""" + }, + ""VerifiedPackages"": { + ""LastModified"": ""2019-01-02T11:00:00+00:00"", + ""LoadDuration"": ""00:00:30"", + ""FileSize"": 5678, + ""ETag"": ""\""etag-b\"""" + }, + ""PopularityTransfers"": { + ""LastModified"": ""2019-01-03T11:00:00+00:00"", + ""LoadDuration"": ""00:00:45"", + ""FileSize"": 9876, + ""ETag"": ""\""etag-c\"""" + } + } +}", actualJson); + Assert.Same(docResult, response.Data[0].Debug); + } + + [Fact] + public void ProducesExpectedResponse() + { + var response = Target.V2FromHijack( + _v2Request, + _text, + _searchParameters, + _hijackResult, + _duration); + + var actualJson = JsonConvert.SerializeObject(response, _jsonSerializerSettings); + Assert.Equal(@"{ + ""totalHits"": 1, + ""data"": [ + { + ""PackageRegistration"": { + ""Id"": ""WindowsAzure.Storage"", + ""DownloadCount"": 1001, + ""Verified"": true, + ""Owners"": [], + ""PopularityTransfers"": [ + ""transfer1"", + ""transfer2"" + ] + }, + ""Version"": ""7.1.2.0-alpha+git"", + ""NormalizedVersion"": ""7.1.2-alpha"", + ""Title"": ""Windows Azure Storage"", + ""Description"": ""Description."", + ""Summary"": ""Summary."", + ""Authors"": ""Microsoft"", + ""Copyright"": ""© Microsoft Corporation. All rights reserved."", + ""Language"": ""en-US"", + ""Tags"": ""Microsoft Azure Storage Table Blob File Queue Scalable windowsazureofficial"", + ""ReleaseNotes"": ""Release notes."", + ""ProjectUrl"": ""https://github.com/Azure/azure-storage-net"", + ""IconUrl"": ""http://go.microsoft.com/fwlink/?LinkID=288890"", + ""IsLatestStable"": false, + ""IsLatest"": true, + ""Listed"": true, + ""Created"": ""2017-01-01T00:00:00+00:00"", + ""Published"": ""2017-01-03T00:00:00+00:00"", + ""LastUpdated"": ""2017-01-03T00:00:00+00:00"", + ""LastEdited"": ""2017-01-02T00:00:00+00:00"", + ""DownloadCount"": 23, + ""FlattenedDependencies"": ""Microsoft.Data.OData:5.6.4:net40-client|Newtonsoft.Json:6.0.8:net40-client"", + ""Dependencies"": [], + ""SupportedFrameworks"": [], + ""MinClientVersion"": ""2.12"", + ""Hash"": ""oMs9XKzRTsbnIpITcqZ5XAv1h2z6oyJ33+Z/PJx36iVikge/8wm5AORqAv7soKND3v5/0QWW9PQ0ktQuQu9aQQ=="", + ""HashAlgorithm"": ""SHA512"", + ""PackageFileSize"": 3039254, + ""LicenseUrl"": ""http://go.microsoft.com/fwlink/?LinkId=331471"", + ""RequiresLicenseAcceptance"": true + } + ] +}", actualJson); + } + + [Fact] + public void UsesFlatContainerUrlWhenConfigured() + { + _config.AllIconsInFlatContainer = true; + + var response = Target.V2FromHijack( + _v2Request, + _text, + _searchParameters, + _hijackResult, + _duration); + + Assert.Equal(Data.FlatContainerIconUrl, response.Data[0].IconUrl); + } + + [Fact] + public void LeavesNullIconUrlWithFlatContainerIconsButNullOriginalIconUrl() + { + _config.AllIconsInFlatContainer = true; + _hijackResult.Results[0].Document.IconUrl = null; + + var response = Target.V2FromHijack( + _v2Request, + _text, + _searchParameters, + _hijackResult, + _duration); + + Assert.Null(response.Data[0].IconUrl); + } + + [Fact] + public void GetsPopularityTransfer() + { + _auxiliaryData + .Setup(x => x.GetPopularityTransfers(It.IsAny())) + .Returns(new[] { "foo", "bar" }); + + var response = Target.V2FromHijack( + _v2Request, + _text, + _searchParameters, + _hijackResult, + _duration); + + var transfers = response.Data[0].PackageRegistration.PopularityTransfers; + Assert.Equal(2, transfers.Length); + Assert.Equal("foo", transfers[0]); + Assert.Equal("bar", transfers[1]); + } + } + + public class V2FromSearch : BaseFacts + { + [Theory] + [MemberData(nameof(MissingTitles))] + public void UsesIdForMissingTitle(string title) + { + _searchResult.Results[0].Document.Title = title; + + var response = Target.V2FromSearch( + _v2Request, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.Equal(Data.PackageId, response.Data[0].Title); + } + + [Fact] + public void CoalescesSomeNullFields() + { + var doc = _searchResult.Results[0].Document; + doc.OriginalVersion = null; + doc.Description = null; + doc.Summary = null; + doc.Authors = null; + doc.Tags = null; + doc.RequiresLicenseAcceptance = null; + + var response = Target.V2FromSearch( + _v2Request, + _text, + _searchParameters, + _searchResult, + _duration); + + var result = response.Data[0]; + Assert.NotNull(result.Version); + Assert.Equal(doc.NormalizedVersion, result.Version); + Assert.Equal(string.Empty, result.Description); + Assert.Equal(string.Empty, result.Summary); + Assert.Equal(string.Empty, result.Authors); + Assert.Equal(string.Empty, result.Tags); + Assert.False(result.RequiresLicenseAcceptance); + } + + [Fact] + public void UsesVersionSpecificDownloadCount() + { + _auxiliaryData.Setup(x => x.GetDownloadCount(It.IsAny(), "7.1.2-alpha")).Returns(4); + + var response = Target.V2FromSearch( + _v2Request, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.Equal(4, response.Data[0].DownloadCount); + } + + [Fact] + public void SortDownloadsAscUsingAuxilaryFile() + { + var mockAuxilaryDataDownloads = new Dictionary(); + for (int i = 0; i < _fiveSearchResults.Count; ++i) + { + var newDoc = _fiveSearchResults.Results[i]; + newDoc.Document.PackageId += i; + newDoc.Document.TotalDownloadCount += i; + newDoc.Document.Title += i; + + // Set download count in reverse order on the Auxilary file + mockAuxilaryDataDownloads.Add(newDoc.Document.PackageId, _fiveSearchResults.Count.Value - i); + } + + var expectedOrder = _fiveSearchResults + .Results + .OrderBy(x => mockAuxilaryDataDownloads[x.Document.PackageId]) + .ThenBy(x => x.Document.Created) + .Select(x => x.Document.PackageId); + + _auxiliaryData + .Setup(x => x.GetTotalDownloadCount(It.IsAny())) + .Returns((string packageId) => + { + return mockAuxilaryDataDownloads[packageId]; + }); + + // Apply the order by ASCENDING TotalDownloadCount and search + _searchParameters.OrderBy = new List { IndexFields.Search.TotalDownloadCount + " asc" }; + _v2Request.SortBy = V2SortBy.TotalDownloadsAsc; + var responseAsc = Target.V2FromSearch( + _v2Request, + _text, + _searchParameters, + _fiveSearchResults, + _duration); + + Assert.Equal(responseAsc.Data.Select(x => x.PackageRegistration.Id), expectedOrder); + } + + [Fact] + public void SortDownloadsDescUsingAuxilaryFile() + { + var mockAuxilaryDataDownloads = new Dictionary(); + for (int i = 0; i < _fiveSearchResults.Count; ++i) + { + var newDoc = _fiveSearchResults.Results[i]; + newDoc.Document.PackageId += i; + newDoc.Document.TotalDownloadCount += i; + newDoc.Document.Title += i; + + // Set download count in reverse order on the Auxilary file + mockAuxilaryDataDownloads.Add(newDoc.Document.PackageId, _fiveSearchResults.Count.Value - i); + } + + var expectedOrder = _fiveSearchResults + .Results + .OrderByDescending(x => mockAuxilaryDataDownloads[x.Document.PackageId]) + .ThenByDescending(x => x.Document.Created) + .Select(x => x.Document.PackageId); + + _auxiliaryData + .Setup(x => x.GetTotalDownloadCount(It.IsAny())) + .Returns((string packageId) => + { + return mockAuxilaryDataDownloads[packageId]; + }); + + // Apply the order by ASCENDING TotalDownloadCount and search + _searchParameters.OrderBy = new List + { + IndexFields.Search.TotalDownloadCount + " desc" + }; + _v2Request.SortBy = V2SortBy.TotalDownloadsDesc; + + var responseDesc = Target.V2FromSearch( + _v2Request, + _text, + _searchParameters, + _fiveSearchResults, + _duration); + + Assert.Equal(responseDesc.Data.Select(x => x.PackageRegistration.Id), expectedOrder); + } + + [Fact] + public void GetsPopularityTransfer() + { + _auxiliaryData + .Setup(x => x.GetPopularityTransfers(It.IsAny())) + .Returns(new[] { "foo", "bar" }); + + var response = Target.V2FromSearch( + _v2Request, + _text, + _searchParameters, + _searchResult, + _duration); + + var transfers = response.Data[0].PackageRegistration.PopularityTransfers; + Assert.Equal(2, transfers.Length); + Assert.Equal("foo", transfers[0]); + Assert.Equal("bar", transfers[1]); + } + + [Fact] + public void CanIncludeDebugInformation() + { + _v2Request.ShowDebug = true; + var docResult = _searchResult.Results[0]; + + var response = Target.V2FromSearch( + _v2Request, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.NotNull(response.Debug); + var actualJson = JsonConvert.SerializeObject(response.Debug, _jsonSerializerSettings); + Assert.Equal(@"{ + ""SearchRequest"": { + ""IgnoreFilter"": false, + ""CountOnly"": false, + ""SortBy"": ""Popularity"", + ""LuceneQuery"": false, + ""Skip"": 0, + ""Take"": 0, + ""IncludePrerelease"": true, + ""IncludeSemVer2"": true, + ""ShowDebug"": true + }, + ""IndexName"": ""search-index"", + ""IndexOperationType"": ""Search"", + ""SearchParameters"": { + ""IncludeTotalResultCount"": false, + ""QueryType"": ""simple"", + ""SearchMode"": ""any"" + }, + ""SearchText"": ""azure storage sdk"", + ""DocumentSearchResult"": { + ""Count"": 1 + }, + ""QueryDuration"": ""00:00:00.2500000"", + ""AuxiliaryFilesMetadata"": { + ""Loaded"": ""2019-01-03T11:00:00+00:00"", + ""Downloads"": { + ""LastModified"": ""2019-01-01T11:00:00+00:00"", + ""LoadDuration"": ""00:00:15"", + ""FileSize"": 1234, + ""ETag"": ""\""etag-a\"""" + }, + ""VerifiedPackages"": { + ""LastModified"": ""2019-01-02T11:00:00+00:00"", + ""LoadDuration"": ""00:00:30"", + ""FileSize"": 5678, + ""ETag"": ""\""etag-b\"""" + }, + ""PopularityTransfers"": { + ""LastModified"": ""2019-01-03T11:00:00+00:00"", + ""LoadDuration"": ""00:00:45"", + ""FileSize"": 9876, + ""ETag"": ""\""etag-c\"""" + } + } +}", actualJson); + Assert.Same(docResult, response.Data[0].Debug); + } + + [Fact] + public void ProducesExpectedResponse() + { + var response = Target.V2FromSearch( + _v2Request, + _text, + _searchParameters, + _searchResult, + _duration); + + var actualJson = JsonConvert.SerializeObject(response, _jsonSerializerSettings); + Assert.Equal(@"{ + ""totalHits"": 1, + ""data"": [ + { + ""PackageRegistration"": { + ""Id"": ""WindowsAzure.Storage"", + ""DownloadCount"": 1001, + ""Verified"": true, + ""Owners"": [ + ""Microsoft"", + ""azure-sdk"" + ], + ""PopularityTransfers"": [ + ""transfer1"", + ""transfer2"" + ] + }, + ""Version"": ""7.1.2.0-alpha+git"", + ""NormalizedVersion"": ""7.1.2-alpha"", + ""Title"": ""Windows Azure Storage"", + ""Description"": ""Description."", + ""Summary"": ""Summary."", + ""Authors"": ""Microsoft"", + ""Copyright"": ""© Microsoft Corporation. All rights reserved."", + ""Language"": ""en-US"", + ""Tags"": ""Microsoft Azure Storage Table Blob File Queue Scalable windowsazureofficial"", + ""ReleaseNotes"": ""Release notes."", + ""ProjectUrl"": ""https://github.com/Azure/azure-storage-net"", + ""IconUrl"": ""http://go.microsoft.com/fwlink/?LinkID=288890"", + ""IsLatestStable"": false, + ""IsLatest"": true, + ""Listed"": true, + ""Created"": ""2017-01-01T00:00:00+00:00"", + ""Published"": ""2017-01-03T00:00:00+00:00"", + ""LastUpdated"": ""2017-01-03T00:00:00+00:00"", + ""LastEdited"": ""2017-01-02T00:00:00+00:00"", + ""DownloadCount"": 23, + ""FlattenedDependencies"": ""Microsoft.Data.OData:5.6.4:net40-client|Newtonsoft.Json:6.0.8:net40-client"", + ""Dependencies"": [], + ""SupportedFrameworks"": [], + ""MinClientVersion"": ""2.12"", + ""Hash"": ""oMs9XKzRTsbnIpITcqZ5XAv1h2z6oyJ33+Z/PJx36iVikge/8wm5AORqAv7soKND3v5/0QWW9PQ0ktQuQu9aQQ=="", + ""HashAlgorithm"": ""SHA512"", + ""PackageFileSize"": 3039254, + ""LicenseUrl"": ""http://go.microsoft.com/fwlink/?LinkId=331471"", + ""RequiresLicenseAcceptance"": true + } + ] +}", actualJson); + } + + [Fact] + public void UsesFlatContainerUrlWhenConfigured() + { + _config.AllIconsInFlatContainer = true; + + var response = Target.V2FromSearch( + _v2Request, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.Equal(Data.FlatContainerIconUrl, response.Data[0].IconUrl); + } + + [Fact] + public void LeavesNullIconUrlWithFlatContainerIconsButNullOriginalIconUrl() + { + _config.AllIconsInFlatContainer = true; + _searchResult.Results[0].Document.IconUrl = null; + + var response = Target.V2FromSearch( + _v2Request, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.Null(response.Data[0].IconUrl); + } + } + + public class V3FromSearch : BaseFacts + { + [Theory] + [InlineData(false, "https://example/reg/")] + [InlineData(true, "https://example/reg-gz-semver2/")] + public void UsesProperSemVer2Url(bool includeSemVer2, string registrationsBaseUrl) + { + _v3Request.IncludeSemVer2 = includeSemVer2; + + var response = Target.V3FromSearch( + _v3Request, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.Equal(response.Context.Base, registrationsBaseUrl); + Assert.Equal(response.Data[0].AtId, registrationsBaseUrl + "windowsazure.storage/index.json"); + Assert.Equal(response.Data[0].Registration, registrationsBaseUrl + "windowsazure.storage/index.json"); + Assert.All( + response.Data[0].Versions, + x => + { + var lowerVersion = NuGetVersion.Parse(x.Version).ToNormalizedString().ToLowerInvariant(); + Assert.Equal(x.AtId, registrationsBaseUrl + "windowsazure.storage/" + lowerVersion + ".json"); + }); + } + + [Theory] + [MemberData(nameof(MissingTitles))] + public void UsesIdForMissingTitle(string title) + { + _searchResult.Results[0].Document.Title = title; + + var response = Target.V3FromSearch( + _v3Request, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.Equal(Data.PackageId, response.Data[0].Title); + } + + [Fact] + public void CoalescesSomeNullFields() + { + var doc = _searchResult.Results[0].Document; + doc.Description = null; + doc.Summary = null; + doc.Tags = null; + doc.Authors = null; + + var response = Target.V3FromSearch( + _v3Request, + _text, + _searchParameters, + _searchResult, + _duration); + + var result = response.Data[0]; + Assert.Equal(string.Empty, result.Description); + Assert.Equal(string.Empty, result.Summary); + Assert.Empty(result.Tags); + Assert.Equal(string.Empty, Assert.Single(result.Authors)); + } + + [Fact] + public void UsesVersionSpecificDownloadCount() + { + _auxiliaryData.Setup(x => x.GetDownloadCount(It.IsAny(), "1.0.0")).Returns(1); + _auxiliaryData.Setup(x => x.GetDownloadCount(It.IsAny(), "2.0.0")).Returns(2); + _auxiliaryData.Setup(x => x.GetDownloadCount(It.IsAny(), "3.0.0-alpha.1")).Returns(3); + _auxiliaryData.Setup(x => x.GetDownloadCount(It.IsAny(), "7.1.2-alpha")).Returns(4); + + var response = Target.V3FromSearch( + _v3Request, + _text, + _searchParameters, + _searchResult, + _duration); + + var versions = response.Data[0].Versions; + Assert.Equal(1, versions[0].Downloads); + Assert.Equal(2, versions[1].Downloads); + Assert.Equal(3, versions[2].Downloads); + Assert.Equal(4, versions[3].Downloads); + } + + [Fact] + public void AllowsNullPackageTypes() + { + var docResult = _searchResult.Results[0]; + docResult.Document.FilterablePackageTypes = null; + docResult.Document.PackageTypes = null; + + var response = Target.V3FromSearch( + _v3Request, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.Null(response.Data[0].PackageTypes); + } + + [Fact] + public void AllowsEmptyPackageTypes() + { + var docResult = _searchResult.Results[0]; + docResult.Document.FilterablePackageTypes = new string[0]; + docResult.Document.PackageTypes = new string[0]; + + var response = Target.V3FromSearch( + _v3Request, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.Null(response.Data[0].PackageTypes); + } + + [Fact] + public void UsesOnlyTheDisplayPackageTypes() + { + var docResult = _searchResult.Results[0]; + docResult.Document.FilterablePackageTypes = new[] { "dependency", "dotnettool" }; + docResult.Document.PackageTypes = null; + + var response = Target.V3FromSearch( + _v3Request, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.Null(response.Data[0].PackageTypes); + } + + [Fact] + public void CanIncludeDebugInformation() + { + _v3Request.ShowDebug = true; + var docResult = _searchResult.Results[0]; + + var response = Target.V3FromSearch( + _v3Request, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.Same(docResult, response.Data[0].Debug); + + Assert.NotNull(response.Debug); + var rootDebugJson = JsonConvert.SerializeObject(response.Debug, _jsonSerializerSettings); + Assert.Equal(@"{ + ""SearchRequest"": { + ""Skip"": 0, + ""Take"": 0, + ""IncludePrerelease"": true, + ""IncludeSemVer2"": true, + ""ShowDebug"": true + }, + ""IndexName"": ""search-index"", + ""IndexOperationType"": ""Search"", + ""SearchParameters"": { + ""IncludeTotalResultCount"": false, + ""QueryType"": ""simple"", + ""SearchMode"": ""any"" + }, + ""SearchText"": ""azure storage sdk"", + ""DocumentSearchResult"": { + ""Count"": 1 + }, + ""QueryDuration"": ""00:00:00.2500000"", + ""AuxiliaryFilesMetadata"": { + ""Loaded"": ""2019-01-03T11:00:00+00:00"", + ""Downloads"": { + ""LastModified"": ""2019-01-01T11:00:00+00:00"", + ""LoadDuration"": ""00:00:15"", + ""FileSize"": 1234, + ""ETag"": ""\""etag-a\"""" + }, + ""VerifiedPackages"": { + ""LastModified"": ""2019-01-02T11:00:00+00:00"", + ""LoadDuration"": ""00:00:30"", + ""FileSize"": 5678, + ""ETag"": ""\""etag-b\"""" + }, + ""PopularityTransfers"": { + ""LastModified"": ""2019-01-03T11:00:00+00:00"", + ""LoadDuration"": ""00:00:45"", + ""FileSize"": 9876, + ""ETag"": ""\""etag-c\"""" + } + } +}", rootDebugJson); + } + + [Fact] + public void ProducesExpectedResponse() + { + var docResult = _searchResult.Results[0]; + docResult.Document.FilterablePackageTypes = new[] { "dependency", "dotnettool" }; + docResult.Document.PackageTypes = new[] { "Dependency", "DotnetTool" }; + + var response = Target.V3FromSearch( + _v3Request, + _text, + _searchParameters, + _searchResult, + _duration); + + var actualJson = JsonConvert.SerializeObject(response, _jsonSerializerSettings); + Assert.Equal(@"{ + ""@context"": { + ""@vocab"": ""http://schema.nuget.org/schema#"", + ""@base"": ""https://example/reg-gz-semver2/"" + }, + ""totalHits"": 1, + ""data"": [ + { + ""@id"": ""https://example/reg-gz-semver2/windowsazure.storage/index.json"", + ""@type"": ""Package"", + ""registration"": ""https://example/reg-gz-semver2/windowsazure.storage/index.json"", + ""id"": ""WindowsAzure.Storage"", + ""version"": ""7.1.2-alpha+git"", + ""description"": ""Description."", + ""summary"": ""Summary."", + ""title"": ""Windows Azure Storage"", + ""iconUrl"": ""http://go.microsoft.com/fwlink/?LinkID=288890"", + ""licenseUrl"": ""http://go.microsoft.com/fwlink/?LinkId=331471"", + ""projectUrl"": ""https://github.com/Azure/azure-storage-net"", + ""tags"": [ + ""Microsoft"", + ""Azure"", + ""Storage"", + ""Table"", + ""Blob"", + ""File"", + ""Queue"", + ""Scalable"", + ""windowsazureofficial"" + ], + ""authors"": [ + ""Microsoft"" + ], + ""totalDownloads"": 1001, + ""verified"": true, + ""packageTypes"": [ + { + ""name"": ""Dependency"" + }, + { + ""name"": ""DotnetTool"" + } + ], + ""versions"": [ + { + ""version"": ""1.0.0"", + ""downloads"": 23, + ""@id"": ""https://example/reg-gz-semver2/windowsazure.storage/1.0.0.json"" + }, + { + ""version"": ""2.0.0+git"", + ""downloads"": 23, + ""@id"": ""https://example/reg-gz-semver2/windowsazure.storage/2.0.0.json"" + }, + { + ""version"": ""3.0.0-alpha.1"", + ""downloads"": 23, + ""@id"": ""https://example/reg-gz-semver2/windowsazure.storage/3.0.0-alpha.1.json"" + }, + { + ""version"": ""7.1.2-alpha+git"", + ""downloads"": 23, + ""@id"": ""https://example/reg-gz-semver2/windowsazure.storage/7.1.2-alpha.json"" + } + ] + } + ] +}", actualJson); + } + + [Fact] + public void UsesFlatContainerUrlWhenConfigured() + { + _config.AllIconsInFlatContainer = true; + + var response = Target.V3FromSearch( + _v3Request, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.Equal(Data.FlatContainerIconUrl, response.Data[0].IconUrl); + } + + [Fact] + public void LeavesNullIconUrlWithFlatContainerIconsButNullOriginalIconUrl() + { + _config.AllIconsInFlatContainer = true; + _searchResult.Results[0].Document.IconUrl = null; + + var response = Target.V3FromSearch( + _v3Request, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.Null(response.Data[0].IconUrl); + } + } + + public class V3FromSearchDocument : BaseFacts + { + [Fact] + public void CanIncludeDebugInformation() + { + _v3Request.ShowDebug = true; + var doc = _searchResult.Results[0].Document; + + var response = Target.V3FromSearchDocument( + _v3Request, + doc.Key, + doc, + _duration); + + var debugDoc = Assert.IsType(response.Data[0].Debug); + Assert.Same(doc, debugDoc.Document); + + Assert.NotNull(response.Debug); + var rootDebugJson = JsonConvert.SerializeObject(response.Debug, _jsonSerializerSettings); + Assert.Equal(@"{ + ""SearchRequest"": { + ""Skip"": 0, + ""Take"": 0, + ""IncludePrerelease"": true, + ""IncludeSemVer2"": true, + ""ShowDebug"": true + }, + ""IndexName"": ""search-index"", + ""IndexOperationType"": ""Get"", + ""DocumentKey"": ""windowsazure_storage-d2luZG93c2F6dXJlLnN0b3JhZ2U1-IncludePrereleaseAndSemVer2"", + ""QueryDuration"": ""00:00:00.2500000"", + ""AuxiliaryFilesMetadata"": { + ""Loaded"": ""2019-01-03T11:00:00+00:00"", + ""Downloads"": { + ""LastModified"": ""2019-01-01T11:00:00+00:00"", + ""LoadDuration"": ""00:00:15"", + ""FileSize"": 1234, + ""ETag"": ""\""etag-a\"""" + }, + ""VerifiedPackages"": { + ""LastModified"": ""2019-01-02T11:00:00+00:00"", + ""LoadDuration"": ""00:00:30"", + ""FileSize"": 5678, + ""ETag"": ""\""etag-b\"""" + }, + ""PopularityTransfers"": { + ""LastModified"": ""2019-01-03T11:00:00+00:00"", + ""LoadDuration"": ""00:00:45"", + ""FileSize"": 9876, + ""ETag"": ""\""etag-c\"""" + } + } +}", rootDebugJson); + } + + [Fact] + public void ProducesExpectedResponse() + { + var doc = _searchResult.Results[0].Document; + + var response = Target.V3FromSearchDocument( + _v3Request, + doc.Key, + doc, + _duration); + + var actualJson = JsonConvert.SerializeObject(response, _jsonSerializerSettings); + Assert.Equal(@"{ + ""@context"": { + ""@vocab"": ""http://schema.nuget.org/schema#"", + ""@base"": ""https://example/reg-gz-semver2/"" + }, + ""totalHits"": 1, + ""data"": [ + { + ""@id"": ""https://example/reg-gz-semver2/windowsazure.storage/index.json"", + ""@type"": ""Package"", + ""registration"": ""https://example/reg-gz-semver2/windowsazure.storage/index.json"", + ""id"": ""WindowsAzure.Storage"", + ""version"": ""7.1.2-alpha+git"", + ""description"": ""Description."", + ""summary"": ""Summary."", + ""title"": ""Windows Azure Storage"", + ""iconUrl"": ""http://go.microsoft.com/fwlink/?LinkID=288890"", + ""licenseUrl"": ""http://go.microsoft.com/fwlink/?LinkId=331471"", + ""projectUrl"": ""https://github.com/Azure/azure-storage-net"", + ""tags"": [ + ""Microsoft"", + ""Azure"", + ""Storage"", + ""Table"", + ""Blob"", + ""File"", + ""Queue"", + ""Scalable"", + ""windowsazureofficial"" + ], + ""authors"": [ + ""Microsoft"" + ], + ""totalDownloads"": 1001, + ""verified"": true, + ""packageTypes"": [ + { + ""name"": ""Dependency"" + } + ], + ""versions"": [ + { + ""version"": ""1.0.0"", + ""downloads"": 23, + ""@id"": ""https://example/reg-gz-semver2/windowsazure.storage/1.0.0.json"" + }, + { + ""version"": ""2.0.0+git"", + ""downloads"": 23, + ""@id"": ""https://example/reg-gz-semver2/windowsazure.storage/2.0.0.json"" + }, + { + ""version"": ""3.0.0-alpha.1"", + ""downloads"": 23, + ""@id"": ""https://example/reg-gz-semver2/windowsazure.storage/3.0.0-alpha.1.json"" + }, + { + ""version"": ""7.1.2-alpha+git"", + ""downloads"": 23, + ""@id"": ""https://example/reg-gz-semver2/windowsazure.storage/7.1.2-alpha.json"" + } + ] + } + ] +}", actualJson); + } + + [Fact] + public void UsesFlatContainerUrlWhenConfigured() + { + _config.AllIconsInFlatContainer = true; + var doc = _searchResult.Results[0].Document; + + var response = Target.V3FromSearchDocument( + _v3Request, + doc.Key, + doc, + _duration); + + Assert.Equal(Data.FlatContainerIconUrl, response.Data[0].IconUrl); + } + + [Fact] + public void LeavesNullIconUrlWithFlatContainerIconsButNullOriginalIconUrl() + { + _config.AllIconsInFlatContainer = true; + var doc = _searchResult.Results[0].Document; + doc.IconUrl = null; + + var response = Target.V3FromSearchDocument( + _v3Request, + doc.Key, + doc, + _duration); + + Assert.Null(response.Data[0].IconUrl); + } + } + + public class AutocompleteFromSearch : BaseFacts + { + [Fact] + public void CanIncludeDebugInformation() + { + _autocompleteRequest.ShowDebug = true; + + var response = Target.AutocompleteFromSearch( + _autocompleteRequest, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.NotNull(response.Debug); + var actualJson = JsonConvert.SerializeObject(response.Debug, _jsonSerializerSettings); + Assert.Equal(@"{ + ""SearchRequest"": { + ""Type"": ""PackageIds"", + ""Skip"": 0, + ""Take"": 0, + ""IncludePrerelease"": true, + ""IncludeSemVer2"": true, + ""ShowDebug"": true + }, + ""IndexName"": ""search-index"", + ""IndexOperationType"": ""Search"", + ""SearchParameters"": { + ""IncludeTotalResultCount"": false, + ""QueryType"": ""simple"", + ""SearchMode"": ""any"" + }, + ""SearchText"": ""azure storage sdk"", + ""DocumentSearchResult"": { + ""Count"": 1 + }, + ""QueryDuration"": ""00:00:00.2500000"" +}", actualJson); + } + + [Fact] + public void ReturnsPackageIds() + { + _autocompleteRequest.Type = AutocompleteRequestType.PackageIds; + + var response = Target.AutocompleteFromSearch( + _autocompleteRequest, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.NotNull(response); + Assert.Single(response.Data); + Assert.Equal("WindowsAzure.Storage", response.Data[0]); + } + + [Fact] + public void ReturnsEmptyPackageVersions() + { + _autocompleteRequest.Type = AutocompleteRequestType.PackageVersions; + + var response = Target.AutocompleteFromSearch( + _autocompleteRequest, + _text, + _searchParameters, + _emptySearchResult, + _duration); + + Assert.NotNull(response); + Assert.Empty(response.Data); + } + + [Fact] + public void ReturnsPackageVersions() + { + _autocompleteRequest.Type = AutocompleteRequestType.PackageVersions; + + var response = Target.AutocompleteFromSearch( + _autocompleteRequest, + _text, + _searchParameters, + _searchResult, + _duration); + + Assert.NotNull(response); + Assert.Equal(4, response.Data.Count); + Assert.Equal("1.0.0", response.Data[0]); + Assert.Equal("2.0.0+git", response.Data[1]); + Assert.Equal("3.0.0-alpha.1", response.Data[2]); + Assert.Equal("7.1.2-alpha+git", response.Data[3]); + } + + [Fact] + public void PackageVersionsThrowsIfMultipleResults() + { + _autocompleteRequest.Type = AutocompleteRequestType.PackageVersions; + + var exception = Assert.Throws(() => Target.AutocompleteFromSearch( + _autocompleteRequest, + _text, + _searchParameters, + _manySearchResults, + _duration)); + + Assert.Equal("result", exception.ParamName); + Assert.Contains("Package version autocomplete queries should have a single document result", exception.Message); + } + } + + public class EmptyV3 : BaseFacts + { + [Fact] + public void ProducesExpectedResponse() + { + var response = Target.EmptyV3(_v3Request); + + var actualJson = JsonConvert.SerializeObject(response, _jsonSerializerSettings); + Assert.Equal(@"{ + ""@context"": { + ""@vocab"": ""http://schema.nuget.org/schema#"", + ""@base"": ""https://example/reg-gz-semver2/"" + }, + ""totalHits"": 0, + ""data"": [] +}", actualJson); + } + + [Fact] + public void CanIncludeDebugInformation() + { + _v3Request.ShowDebug = true; + + var response = Target.EmptyV3(_v3Request); + + Assert.NotNull(response.Debug); + var actualJson = JsonConvert.SerializeObject(response.Debug, _jsonSerializerSettings); + Assert.Equal(@"{ + ""SearchRequest"": { + ""Skip"": 0, + ""Take"": 0, + ""IncludePrerelease"": true, + ""IncludeSemVer2"": true, + ""ShowDebug"": true + }, + ""IndexOperationType"": ""Empty"" +}", actualJson); + } + } + + public class EmptyV2 : BaseFacts + { + [Fact] + public void ProducesExpectedResponse() + { + var response = Target.EmptyV2(_v2Request); + + var actualJson = JsonConvert.SerializeObject(response, _jsonSerializerSettings); + Assert.Equal(@"{ + ""totalHits"": 0, + ""data"": [] +}", actualJson); + } + + [Fact] + public void CanIncludeDebugInformation() + { + _v2Request.ShowDebug = true; + + var response = Target.EmptyV2(_v2Request); + + Assert.NotNull(response.Debug); + var actualJson = JsonConvert.SerializeObject(response.Debug, _jsonSerializerSettings); + Assert.Equal(@"{ + ""SearchRequest"": { + ""IgnoreFilter"": false, + ""CountOnly"": false, + ""SortBy"": ""Popularity"", + ""LuceneQuery"": false, + ""Skip"": 0, + ""Take"": 0, + ""IncludePrerelease"": true, + ""IncludeSemVer2"": true, + ""ShowDebug"": true + }, + ""IndexOperationType"": ""Empty"" +}", actualJson); + } + } + + public class EmptyAutocomplete : BaseFacts + { + [Fact] + public void ProducesExpectedResponse() + { + var response = Target.EmptyAutocomplete(_autocompleteRequest); + + var actualJson = JsonConvert.SerializeObject(response, _jsonSerializerSettings); + Assert.Equal(@"{ + ""@context"": { + ""@vocab"": ""http://schema.nuget.org/schema#"" + }, + ""totalHits"": 0, + ""data"": [] +}", actualJson); + } + + [Fact] + public void CanIncludeDebugInformation() + { + _autocompleteRequest.ShowDebug = true; + + var response = Target.EmptyAutocomplete(_autocompleteRequest); + + Assert.NotNull(response.Debug); + var actualJson = JsonConvert.SerializeObject(response.Debug, _jsonSerializerSettings); + Assert.Equal(@"{ + ""SearchRequest"": { + ""Type"": ""PackageIds"", + ""Skip"": 0, + ""Take"": 0, + ""IncludePrerelease"": true, + ""IncludeSemVer2"": true, + ""ShowDebug"": true + }, + ""IndexOperationType"": ""Empty"" +}", actualJson); + } + } + + public abstract class BaseFacts + { + protected readonly Mock _auxiliaryData; + protected readonly SearchServiceConfiguration _config; + protected readonly Mock> _options; + protected readonly V2SearchRequest _v2Request; + protected readonly V3SearchRequest _v3Request; + protected readonly AutocompleteRequest _autocompleteRequest; + protected readonly SearchParameters _searchParameters; + protected readonly string _text; + protected readonly TimeSpan _duration; + protected readonly DocumentSearchResult _searchResult; + protected readonly DocumentSearchResult _fiveSearchResults; + protected readonly DocumentSearchResult _emptySearchResult; + protected readonly DocumentSearchResult _manySearchResults; + protected readonly DocumentSearchResult _hijackResult; + protected readonly JsonSerializerSettings _jsonSerializerSettings; + protected readonly AuxiliaryFilesMetadata _auxiliaryMetadata; + + public SearchResponseBuilder Target => new SearchResponseBuilder( + new Lazy(() => _auxiliaryData.Object), + _options.Object); + + public static IEnumerable MissingTitles = new[] + { + new object[] { null }, + new object[] { string.Empty }, + new object[] { " " }, + new object[] { " \t"}, + }; + + public BaseFacts() + { + _auxiliaryData = new Mock(); + _config = new SearchServiceConfiguration(); + _options = new Mock>(); + _options.Setup(x => x.Value).Returns(() => _config); + _auxiliaryMetadata = new AuxiliaryFilesMetadata( + new DateTimeOffset(2019, 1, 3, 11, 0, 0, TimeSpan.Zero), + new AuxiliaryFileMetadata( + new DateTimeOffset(2019, 1, 1, 11, 0, 0, TimeSpan.Zero), + TimeSpan.FromSeconds(15), + 1234, + "\"etag-a\""), + new AuxiliaryFileMetadata( + new DateTimeOffset(2019, 1, 2, 11, 0, 0, TimeSpan.Zero), + TimeSpan.FromSeconds(30), + 5678, + "\"etag-b\""), + new AuxiliaryFileMetadata( + new DateTimeOffset(2019, 1, 3, 11, 0, 0, TimeSpan.Zero), + TimeSpan.FromSeconds(45), + 9876, + "\"etag-c\"")); + + _config.SearchIndexName = "search-index"; + _config.HijackIndexName = "hijack-index"; + _config.SemVer1RegistrationsBaseUrl = "https://example/reg/"; + _config.SemVer2RegistrationsBaseUrl = "https://example/reg-gz-semver2/"; + _config.FlatContainerBaseUrl = Data.FlatContainerBaseUrl; + _config.FlatContainerContainerName = Data.FlatContainerContainerName; + + _auxiliaryData + .Setup(x => x.GetTotalDownloadCount(It.IsAny())) + .Returns(Data.TotalDownloadCount); + _auxiliaryData + .Setup(x => x.GetDownloadCount(It.IsAny(), It.IsAny())) + .Returns(23); + _auxiliaryData + .Setup(x => x.IsVerified(It.IsAny())) + .Returns(true); + _auxiliaryData + .Setup(x => x.Metadata) + .Returns(() => _auxiliaryMetadata); + _auxiliaryData + .Setup(x => x.GetPopularityTransfers(It.IsAny())) + .Returns(() => new[] { "transfer1", "transfer2" }); + + _v2Request = new V2SearchRequest + { + IncludePrerelease = true, + IncludeSemVer2 = true, + }; + _v3Request = new V3SearchRequest + { + IncludePrerelease = true, + IncludeSemVer2 = true + }; + _autocompleteRequest = new AutocompleteRequest + { + IncludePrerelease = true, + IncludeSemVer2 = true, + }; + _searchParameters = new SearchParameters(); + _text = "azure storage sdk"; + _duration = TimeSpan.FromMilliseconds(250); + _emptySearchResult = new DocumentSearchResult + { + Count = 0, + Results = new List>(), + }; + _searchResult = new DocumentSearchResult + { + Count = 1, + Results = new List> + { + new SearchResult + { + Document = Data.SearchDocument, + }, + }, + }; + _fiveSearchResults = new DocumentSearchResult + { + Count = 5, + Results = new List> + { + new SearchResult + { + Document = Data.SearchDocument, + }, + new SearchResult + { + Document = Data.SearchDocument, + }, + new SearchResult + { + Document = Data.SearchDocument, + }, + new SearchResult + { + Document = Data.SearchDocument, + }, + new SearchResult + { + Document = Data.SearchDocument, + }, + }, + }; + _manySearchResults = new DocumentSearchResult + { + Count = 2, + Results = new List> + { + new SearchResult + { + Document = Data.SearchDocument, + }, + new SearchResult + { + Document = Data.SearchDocument, + }, + }, + }; + _hijackResult = new DocumentSearchResult + { + Count = 1, + Results = new List> + { + new SearchResult + { + Document = Data.HijackDocument, + }, + }, + }; + + _jsonSerializerSettings = new JsonSerializerSettings + { + NullValueHandling = NullValueHandling.Ignore, + Converters = + { + new StringEnumConverter(), + }, + Formatting = Formatting.Indented, + }; + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchStatusServiceFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchStatusServiceFacts.cs new file mode 100644 index 000000000..469551585 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchStatusServiceFacts.cs @@ -0,0 +1,325 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Search.Models; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using NuGet.Services.AzureSearch.Wrappers; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class SearchStatusServiceFacts + { + public class GetStatusAsync : BaseFacts + { + public GetStatusAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task AllowsSkippingAzureSearch() + { + var status = await _target.GetStatusAsync(SearchStatusOptions.All ^ SearchStatusOptions.AzureSearch, _assembly); + + Assert.True(status.Success); + _searchDocuments.Verify(x => x.CountAsync(), Times.Never); + _searchDocuments.Verify(x => x.SearchAsync(It.IsAny(), It.IsAny()), Times.Never); + _hijackDocuments.Verify(x => x.CountAsync(), Times.Never); + _hijackDocuments.Verify(x => x.SearchAsync(It.IsAny(), It.IsAny()), Times.Never); + Assert.Null(status.SearchIndex); + Assert.Null(status.HijackIndex); + Assert.NotNull(status.Server); + Assert.NotNull(status.AuxiliaryFiles); + } + + [Fact] + public async Task AllowsSkippingAuxiliaryFiles() + { + var status = await _target.GetStatusAsync(SearchStatusOptions.All ^ SearchStatusOptions.AuxiliaryFiles, _assembly); + + Assert.True(status.Success); + _auxiliaryDataCache.Verify(x => x.EnsureInitializedAsync(), Times.Never); + _auxiliaryDataCache.Verify(x => x.Get(), Times.Never); + Assert.Null(status.AuxiliaryFiles); + Assert.NotNull(status.Server); + Assert.NotNull(status.SearchIndex); + Assert.NotNull(status.HijackIndex); + } + + [Fact] + public async Task AllowsSkippingServer() + { + var status = await _target.GetStatusAsync(SearchStatusOptions.All ^ SearchStatusOptions.Server, _assembly); + + Assert.True(status.Success); + Assert.Null(status.Server); + Assert.NotNull(status.AuxiliaryFiles); + Assert.NotNull(status.SearchIndex); + Assert.NotNull(status.HijackIndex); + } + + [Fact] + public async Task ReturnsNullCommitTimestampWhenThereAreNoDocuments() + { + _searchDocuments + .Setup(x => x.SearchAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new DocumentSearchResult()); + + var status = await _target.GetStatusAsync(SearchStatusOptions.All, _assembly); + + Assert.True(status.Success); + Assert.NotNull(status.SearchIndex); + Assert.Null(status.SearchIndex.LastCommitTimestamp); + } + + [Fact] + public async Task ReturnsNullCommitTimestampWhenTheLastCommitTimestampIsNull() + { + _searchDocuments + .Setup(x => x.SearchAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(GetLastCommitTimestampResult(timestamp: null)); + + var status = await _target.GetStatusAsync(SearchStatusOptions.All, _assembly); + + Assert.True(status.Success); + Assert.NotNull(status.SearchIndex); + Assert.Null(status.SearchIndex.LastCommitTimestamp); + } + + [Fact] + public async Task ReturnsFullStatus() + { + var before = DateTimeOffset.UtcNow; + var status = await _target.GetStatusAsync(SearchStatusOptions.All, _assembly); + + Assert.True(status.Success); + Assert.InRange(status.Duration.Value, TimeSpan.FromMilliseconds(1), TimeSpan.MaxValue); + + Assert.Equal("This is a fake build date for testing.", status.Server.AssemblyBuildDateUtc); + Assert.Equal("This is a fake commit ID for testing.", status.Server.AssemblyCommitId); + Assert.Equal("1.0.0-fakefortesting", status.Server.AssemblyInformationalVersion); + Assert.Equal(_config.DeploymentLabel, status.Server.DeploymentLabel); + Assert.Equal("Fake website instance ID.", status.Server.InstanceId); + Assert.Equal(Environment.MachineName, status.Server.MachineName); + Assert.InRange(status.Server.ProcessDuration, TimeSpan.FromMilliseconds(1), TimeSpan.MaxValue); + Assert.NotEqual(0, status.Server.ProcessId); + Assert.InRange(status.Server.ProcessStartTime, DateTimeOffset.MinValue, before); + Assert.Equal(_lastSecretRefresh, status.Server.LastServiceRefreshTime); + + Assert.Equal(23, status.SearchIndex.DocumentCount); + Assert.Equal("search-index", status.SearchIndex.Name); + Assert.InRange(status.SearchIndex.WarmQueryDuration, TimeSpan.FromMilliseconds(1), TimeSpan.MaxValue); + Assert.Equal(_searchLastCommitTimestamp, status.SearchIndex.LastCommitTimestamp); + + Assert.Equal(42, status.HijackIndex.DocumentCount); + Assert.Equal("hijack-index", status.HijackIndex.Name); + Assert.InRange(status.HijackIndex.WarmQueryDuration, TimeSpan.FromMilliseconds(1), TimeSpan.MaxValue); + Assert.Equal(_hijackLastCommitTimestamp, status.HijackIndex.LastCommitTimestamp); + + Assert.Same(_auxiliaryFilesMetadata, status.AuxiliaryFiles); + + _searchDocuments.Verify(x => x.CountAsync(), Times.Once); + _searchDocuments.Verify(x => x.SearchAsync("*", It.IsAny()), Times.Exactly(2)); + _hijackDocuments.Verify(x => x.CountAsync(), Times.Once); + _hijackDocuments.Verify(x => x.SearchAsync("*", It.IsAny()), Times.Exactly(2)); + } + + [Fact] + public async Task HandlesFailedServerStatus() + { + _options.Setup(x => x.Value).Throws(new InvalidOperationException("Can't get the deployment label.")); + + var status = await _target.GetStatusAsync(SearchStatusOptions.All, _assembly); + + Assert.False(status.Success); + Assert.InRange(status.Duration.Value, TimeSpan.FromMilliseconds(1), TimeSpan.MaxValue); + Assert.Null(status.Server); + Assert.NotNull(status.SearchIndex); + Assert.NotNull(status.HijackIndex); + Assert.NotNull(status.AuxiliaryFiles); + } + + [Fact] + public async Task HandlesFailedSearchIndexStatus() + { + _searchDocuments + .Setup(x => x.SearchAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new InvalidOperationException("Could not hit the search index.")); + + var status = await _target.GetStatusAsync(SearchStatusOptions.All, _assembly); + + Assert.False(status.Success); + Assert.InRange(status.Duration.Value, TimeSpan.FromMilliseconds(1), TimeSpan.MaxValue); + Assert.NotNull(status.Server); + Assert.Null(status.SearchIndex); + Assert.NotNull(status.HijackIndex); + Assert.NotNull(status.AuxiliaryFiles); + } + + [Fact] + public async Task HandlesFailedHijackIndexStatus() + { + _hijackDocuments + .Setup(x => x.SearchAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new InvalidOperationException("Could not hit the hijack index.")); + + var status = await _target.GetStatusAsync(SearchStatusOptions.All, _assembly); + + Assert.False(status.Success); + Assert.InRange(status.Duration.Value, TimeSpan.FromMilliseconds(1), TimeSpan.MaxValue); + Assert.NotNull(status.Server); + Assert.NotNull(status.SearchIndex); + Assert.Null(status.HijackIndex); + Assert.NotNull(status.AuxiliaryFiles); + } + + [Fact] + public async Task HandlesFailedAuxiliaryData() + { + _auxiliaryDataCache + .Setup(x => x.EnsureInitializedAsync()) + .Throws(new InvalidOperationException("Could not initialize the auxiliary data.")); + + var status = await _target.GetStatusAsync(SearchStatusOptions.All, _assembly); + + Assert.False(status.Success); + Assert.InRange(status.Duration.Value, TimeSpan.FromMilliseconds(1), TimeSpan.MaxValue); + Assert.NotNull(status.Server); + Assert.NotNull(status.SearchIndex); + Assert.NotNull(status.HijackIndex); + Assert.Null(status.AuxiliaryFiles); + } + } + + public abstract class BaseFacts + { + protected readonly Mock _searchIndex; + protected readonly Mock _searchDocuments; + protected readonly Mock _hijackIndex; + protected readonly Mock _hijackDocuments; + protected readonly Mock _parametersBuilder; + protected readonly Mock _auxiliaryDataCache; + protected readonly Mock _auxiliaryData; + protected readonly Mock _secretRefresher; + protected readonly SearchServiceConfiguration _config; + protected readonly Mock> _options; + protected readonly Mock _telemetryService; + protected readonly RecordingLogger _logger; + protected readonly AuxiliaryFilesMetadata _auxiliaryFilesMetadata; + protected readonly Assembly _assembly; + protected readonly SearchStatusService _target; + protected readonly SearchParameters _lastCommitTimestampParameters; + protected readonly DateTimeOffset _searchLastCommitTimestamp; + protected readonly DateTimeOffset _hijackLastCommitTimestamp; + protected readonly DateTimeOffset _lastSecretRefresh; + + public BaseFacts(ITestOutputHelper output) + { + _searchIndex = new Mock(); + _searchDocuments = new Mock(); + _hijackIndex = new Mock(); + _hijackDocuments = new Mock(); + _parametersBuilder = new Mock(); + _auxiliaryDataCache = new Mock(); + _auxiliaryData = new Mock(); + _secretRefresher = new Mock(); + _options = new Mock>(); + _telemetryService = new Mock(); + _logger = output.GetLogger(); + + _auxiliaryFilesMetadata = new AuxiliaryFilesMetadata( + DateTimeOffset.MinValue, + new AuxiliaryFileMetadata( + DateTimeOffset.MinValue, + TimeSpan.Zero, + 0, + string.Empty), + new AuxiliaryFileMetadata( + DateTimeOffset.MinValue, + TimeSpan.Zero, + 0, + string.Empty), + new AuxiliaryFileMetadata( + DateTimeOffset.MinValue, + TimeSpan.Zero, + 0, + string.Empty)); + _assembly = typeof(BaseFacts).Assembly; + _config = new SearchServiceConfiguration(); + _config.DeploymentLabel = "Fake deployment label."; + _lastCommitTimestampParameters = new SearchParameters(); + _searchLastCommitTimestamp = new DateTimeOffset(2019, 7, 1, 0, 0, 0, TimeSpan.Zero); + _hijackLastCommitTimestamp = new DateTimeOffset(2019, 7, 2, 0, 0, 0, TimeSpan.Zero); + _lastSecretRefresh = new DateTimeOffset(2019, 7, 3, 0, 0, 0, TimeSpan.Zero); + Environment.SetEnvironmentVariable("WEBSITE_INSTANCE_ID", "Fake website instance ID."); + + _secretRefresher.Setup(x => x.LastRefresh).Returns(() => _lastSecretRefresh); + _searchIndex.Setup(x => x.IndexName).Returns("search-index"); + _hijackIndex.Setup(x => x.IndexName).Returns("hijack-index"); + _searchIndex.Setup(x => x.Documents).Returns(() => _searchDocuments.Object); + _hijackIndex.Setup(x => x.Documents).Returns(() => _hijackDocuments.Object); + _searchDocuments.Setup(x => x.CountAsync()).ReturnsAsync(23); + _hijackDocuments.Setup(x => x.CountAsync()).ReturnsAsync(42); + _parametersBuilder.Setup(x => x.LastCommitTimestamp()).Returns(() => _lastCommitTimestampParameters); + + _searchDocuments + .Setup(x => x.SearchAsync(It.IsAny(), It.Is(p => !IsLastCommitTimestamp(p)))) + .ReturnsAsync(new DocumentSearchResult()) + .Callback(() => Thread.Sleep(TimeSpan.FromMilliseconds(1))); + _searchDocuments + .Setup(x => x.SearchAsync(It.IsAny(), It.Is(p => IsLastCommitTimestamp(p)))) + .ReturnsAsync(GetLastCommitTimestampResult(_searchLastCommitTimestamp)); + _hijackDocuments + .Setup(x => x.SearchAsync(It.IsAny(), It.Is(p => !IsLastCommitTimestamp(p)))) + .ReturnsAsync(new DocumentSearchResult()) + .Callback(() => Thread.Sleep(TimeSpan.FromMilliseconds(1))); + _hijackDocuments + .Setup(x => x.SearchAsync(It.IsAny(), It.Is(p => IsLastCommitTimestamp(p)))) + .ReturnsAsync(GetLastCommitTimestampResult(_hijackLastCommitTimestamp)); + _options.Setup(x => x.Value).Returns(() => _config); + _auxiliaryDataCache.Setup(x => x.Get()).Returns(() => _auxiliaryData.Object); + _auxiliaryData.Setup(x => x.Metadata).Returns(() => _auxiliaryFilesMetadata); + + _target = new SearchStatusService( + _searchIndex.Object, + _hijackIndex.Object, + _parametersBuilder.Object, + _auxiliaryDataCache.Object, + _secretRefresher.Object, + _options.Object, + _telemetryService.Object, + _logger); + } + + protected static DocumentSearchResult GetLastCommitTimestampResult(DateTimeOffset? timestamp) + { + return new DocumentSearchResult + { + Results = new List + { + new SearchResult + { + Document = new Document + { + { "lastCommitTimestamp", timestamp }, + }, + }, + }, + }; + } + + private bool IsLastCommitTimestamp(SearchParameters parameters) + { + return parameters == _lastCommitTimestampParameters; + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchTextBuilderFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchTextBuilderFacts.cs new file mode 100644 index 000000000..c60da07db --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchService/SearchTextBuilderFacts.cs @@ -0,0 +1,418 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Options; +using Moq; +using Xunit; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class SearchTextBuilderFacts + { + public class V2Search : FactsBase + { + [Theory] + [MemberData(nameof(CommonAzureSearchQueryData))] + public void GeneratesAzureSearchQuery(string input, string expected) + { + var parsed = _target.ParseV2Search(new V2SearchRequest { Query = input }); + + var actual = _target.Build(parsed); + + Assert.Equal(expected, actual); + } + + [Theory] + [InlineData(false, "tokenizedPackageId:hello")] + [InlineData(true, "packageId:hello")] + public void WhenLuceneQuery_TreatsLeadingIdAsPackageId(bool luceneQuery, string expected) + { + var parsed = _target.ParseV2Search(new V2SearchRequest + { + Query = "id:hello", + LuceneQuery = luceneQuery, + }); + + var actual = _target.Build(parsed); + + Assert.Equal(expected, actual); + } + + [Theory] + [MemberData(nameof(GenerateQueriesManyClauses))] + public void ThrowsWhenQueryHasTooManyClauses(int nonFieldScopedTerms, int fieldScopedTerms, bool shouldThrow) + { + var request = new V2SearchRequest { Query = GenerateQuery(nonFieldScopedTerms, fieldScopedTerms) }; + var parsed = _target.ParseV2Search(request); + + if (shouldThrow) + { + var e = Assert.Throws(() => _target.Build(parsed)); + Assert.Equal("A query can only have up to 1024 clauses.", e.Message); + } + else + { + _target.ParseV2Search(request); + } + } + + [Theory] + [MemberData(nameof(GenerateQueryWithTooBigTerm))] + public void ThrowsWhenTermIsTooBig(string query) + { + var request = new V2SearchRequest { Query = query }; + var parsed = _target.ParseV2Search(request); + + var e = Assert.Throws(() => _target.Build(parsed)); + + Assert.Equal("Query terms cannot exceed 32768 bytes.", e.Message); + } + } + + public class V3Search : FactsBase + { + [Theory] + [MemberData(nameof(CommonAzureSearchQueryData))] + public void GeneratesAzureSearchQuery(string input, string expected) + { + var parsed = _target.ParseV3Search(new V3SearchRequest { Query = input }); + + var actual = _target.Build(parsed); + + Assert.Equal(expected, actual); + } + + [Theory] + [MemberData(nameof(GenerateQueriesManyClauses))] + public void ThrowsWhenQueryHasTooManyClauses(int nonFieldScopedTerms, int fieldScopedTerms, bool shouldThrow) + { + var request = new V3SearchRequest { Query = GenerateQuery(nonFieldScopedTerms, fieldScopedTerms) }; + var parsed = _target.ParseV3Search(request); + + if (shouldThrow) + { + var e = Assert.Throws(() => _target.Build(parsed)); + Assert.Equal("A query can only have up to 1024 clauses.", e.Message); + } + else + { + _target.ParseV3Search(request); + } + } + + [Theory] + [MemberData(nameof(GenerateQueryWithTooBigTerm))] + public void ThrowsWhenTermIsTooBig(string query) + { + var request = new V3SearchRequest { Query = query }; + var parsed = _target.ParseV3Search(request); + + var e = Assert.Throws(() => _target.Build(parsed)); + + Assert.Equal("Query terms cannot exceed 32768 bytes.", e.Message); + } + } + + public class Autocomplete : FactsBase + { + [Theory] + [InlineData("Test", "packageId:Test* +tokenizedPackageId:Test* packageId:Test^1000")] + [InlineData("Test ", "packageId:Test* +tokenizedPackageId:Test* packageId:Test^1000")] + [InlineData("title:test", "packageId:title\\:test* +tokenizedPackageId:title\\:test*")] + [InlineData("Hello world", "packageId:Hello\\ world* +tokenizedPackageId:Hello\\ world*")] + [InlineData("Hello world ", "packageId:Hello\\ world* +tokenizedPackageId:Hello\\ world*")] + [InlineData("Hello.world", "packageId:Hello.world* +tokenizedPackageId:Hello* +tokenizedPackageId:world* packageId:Hello.world^1000")] + [InlineData("Foo.BarBaz", "packageId:Foo.BarBaz* +tokenizedPackageId:Foo* +tokenizedPackageId:BarBaz* packageId:Foo.BarBaz^1000")] + public void PackageIdAutocomplete(string input, string expected) + { + var request = new AutocompleteRequest + { + Query = input, + Type = AutocompleteRequestType.PackageIds + }; + + var actual = _target.Autocomplete(request); + + Assert.Equal(expected, actual); + } + + [Theory] + [InlineData("Test", "packageId:Test")] + [InlineData("Hello world", @"packageId:""Hello world""")] + public void PackageVersionsAutocomplete(string input, string expected) + { + var request = new AutocompleteRequest + { + Query = input, + Type = AutocompleteRequestType.PackageVersions + }; + + var actual = _target.Autocomplete(request); + + Assert.Equal(expected, actual); + } + } + + public class FactsBase + { + protected readonly SearchTextBuilder _target; + + public FactsBase() + { + var config = new SearchServiceConfiguration { MatchAllTermsBoost = 2.0f }; + var options = new Mock>(); + options.Setup(o => o.Value).Returns(config); + + _target = new SearchTextBuilder(options.Object); + } + + public static IEnumerable CommonAzureSearchQueryData() + { + // Map of inputs to expected output + var data = new Dictionary + { + { "", "*" }, + { " ", "*" }, + + { "id:test", "tokenizedPackageId:test" }, + { "packageId:json", "packageId:json" }, + { "version:1.0.0-test", "normalizedVersion:1.0.0\\-test" }, + { "title:hello", "title:hello" }, + { "description:hello", "description:hello" }, + { "tag:hi", "tags:hi" }, + { "tags:foo", "tags:foo" }, + { "author:bob", "authors:bob" }, + { "authors:billy", "authors:billy" }, + { "summary:test", "summary:test" }, + { "owner:goat", "owners:goat" }, + { "owners:nugget", "owners:nugget" }, + + // The NuGet query fields are case insensitive + { "ID:TEST", "tokenizedPackageId:TEST" }, + { "PACKAGEID:JSON", "packageId:JSON" }, + { "VERSION:1.0.0-TEST", "normalizedVersion:1.0.0\\-TEST" }, + { "TITLE:HELLO", "title:HELLO" }, + { "DESCRIPTION:HELLO", "description:HELLO" }, + { "TAG:HI", "tags:HI" }, + { "TAGS:FOO", "tags:FOO" }, + { "AUTHOR:BOB", "authors:BOB" }, + { "AUTHORS:BILLY", "authors:BILLY" }, + { "SUMMARY:TEST", "summary:TEST" }, + { "OWNER:GOAT", "owners:GOAT" }, + { "OWNERS:NUGGET", "owners:NUGGET" }, + + // Unknown fields are ignored + { "fake:test", "*" }, + { "foo:a bar:b", "*" }, + + // The version field is normalized, if possible + { "version:1.0.0.0", "normalizedVersion:1.0.0" }, + { "version:1.0.0.0-test", "normalizedVersion:1.0.0\\-test" }, + { "version:Thisisnotavalidversion", "normalizedVersion:Thisisnotavalidversion" }, + + // The tags field is split by delimiters + { "tag:a,b;c|d", "tags:(a b c d)" }, + { "tags:a,b;c|d", "tags:(a b c d)" }, + { "tags:,;|", "*" }, + + { "id:foo id:bar", "tokenizedPackageId:(foo bar)" }, + { "packageId:foo packageId:bar", "packageId:(foo bar)" }, + { "title:hello title:world", "title:(hello world)" }, + { "description:I description:am", "description:(I am)" }, + { "tag:a tag:sentient tags:being", "tags:(a sentient being)" }, + { "author:a author:b authors:c", "authors:(a b c)" }, + { "summary:d summary:e", "summary:(d e)" }, + { "owner:billy owners:the owner:goat", "owners:(billy the goat)" }, + { @"tag:a,b;c tags:d tags:""e f""", "tags:(a b c d e f)" }, + + // If there are multiple terms, each field-scoped term must have at least one match + { "title:foo description:bar title:baz", "+title:(foo baz) +description:bar" }, + { "title:foo bar", "+title:foo bar" }, + { "title:foo unknown:bar", "title:foo" }, + + // If there are non-field-scoped terms and no field-scoped terms, at least of one the non-field-scoped terms is required. + // If there are no field-scoped terms, results that prefix match the last term are boosted + // Results that match all terms are boosted. + // Results that match all terms after tokenization are boosted. + { "foo", "foo tokenizedPackageId:foo*^20" }, + { "foobar", "foobar tokenizedPackageId:foobar*" }, + { "foo bar", "foo bar (+foo +bar)^2 tokenizedPackageId:bar*^20" }, + { "foo.bar baz.qux", "foo.bar baz.qux (+foo.bar +baz.qux)^2 (+foo +bar +baz +qux)^2 packageId:baz.qux*^20" }, + { "id packageId VERSION Title description tag author summary owner owners", + "id packageId VERSION Title description tag author summary owner owners " + + "(+id +packageId +VERSION +Title +description +tag +author +summary +owner +owners)^2 " + + "(+id +package +Id +VERSION +Title +description +tag +author +summary +owner +owners)^2 tokenizedPackageId:owners*" }, + + // If there is a single non-field-scoped term that is a valid package ID and has separator + // characters, boost results that match all tokens, boost results that prefix match the last token, + // and mega boost the exact match. + { "foo.bar", "foo.bar (+foo +bar)^2 packageId:foo.bar*^20 packageId:foo.bar^1000" }, + { "foo_bar", "foo_bar (+foo +bar)^2 packageId:foo_bar*^20 packageId:foo_bar^1000" }, + { "foo-bar", @"foo\-bar (+foo +bar)^2 packageId:foo\-bar*^20 packageId:foo\-bar^1000" }, + { " foo.bar.Baz ", "foo.bar.Baz (+foo +bar +Baz)^2 packageId:foo.bar.Baz*^20 packageId:foo.bar.Baz^1000" }, + { @"""foo.bar""", @"foo.bar (+foo +bar)^2 packageId:foo.bar*^20 packageId:foo.bar^1000" }, + { @"""foo-bar""", @"foo\-bar (+foo +bar)^2 packageId:foo\-bar*^20 packageId:foo\-bar^1000" }, + { @"""foo_bar""", @"foo_bar (+foo +bar)^2 packageId:foo_bar*^20 packageId:foo_bar^1000" }, + + // Boost results that match all tokens from unscoped terms in the query. + { "foo.bar buzz", "foo.bar buzz (+foo.bar +buzz)^2 (+foo +bar +buzz)^2 tokenizedPackageId:buzz*" }, + { "foo_bar buzz", "foo_bar buzz (+foo_bar +buzz)^2 (+foo +bar +buzz)^2 tokenizedPackageId:buzz*" }, + { "foo-bar buzz", @"foo\-bar buzz (+foo\-bar +buzz)^2 (+foo +bar +buzz)^2 tokenizedPackageId:buzz*" }, + { "foo,bar, buzz", @"foo,bar, buzz (+foo,bar, +buzz)^2 (+foo +bar +buzz)^2 tokenizedPackageId:buzz*" }, + { "fooBar buzz", "fooBar buzz (+fooBar +buzz)^2 (+foo +Bar +buzz)^2 tokenizedPackageId:buzz*" }, + { "foo5 buzz", "foo5 buzz (+foo5 +buzz)^2 (+foo +5 +buzz)^2 tokenizedPackageId:buzz*" }, + { "FOO5 buzz", "FOO5 buzz (+FOO5 +buzz)^2 (+FOO +5 +buzz)^2 tokenizedPackageId:buzz*" }, + { "5FOO buzz", "5FOO buzz (+5FOO +buzz)^2 (+5 +FOO +buzz)^2 tokenizedPackageId:buzz*" }, + { "foo5foo", "foo5foo (+foo +5)^2 tokenizedPackageId:foo5foo*" }, + { "FOO5FOO", "FOO5FOO (+FOO +5)^2 tokenizedPackageId:FOO5FOO*" }, + { "fooFoo", "fooFoo (+foo +Foo)^2 tokenizedPackageId:fooFoo*" }, + { "FOOFoo", "FOOFoo tokenizedPackageId:FOOFoo*" }, + + // Phrases are supported in queries + { @"""foo bar""", @"""foo bar"" tokenizedPackageId:foo\ bar*" }, + { @"""foo bar"" baz", @"""foo bar"" baz (+""foo bar"" +baz)^2 tokenizedPackageId:baz*^20" }, + { @"title:""foo bar""", @"title:""foo bar""" }, + { @"title:""a b"" c title:d f", @"+title:(""a b"" d) c f (+c +f)^2" }, + { @"title:"" a b c """, @"title:""a b c""" }, + + // Dangling quotes are handled with best effort + { @"Tags:""windows", "tags:windows" }, + { @"json Tags:""net"" Tags:""windows sdk", @"+tags:(net windows sdk) json" }, + { @"json Tags:""net Tags:""windows sdk""", @"+tags:(net Tags\:) json windows sdk (+json +windows +sdk)^2" }, + { @"sdk Tags:""windows", "+tags:windows sdk" }, + { @"Tags:""windows sdk", "tags:(windows sdk)" }, + { @"Tags:""""windows""", "windows tokenizedPackageId:windows*" }, + + // Empty quotes are ignored + { @"Tags:""""", @"*" }, + { @"Tags:"" """, @"*" }, + { @"Tags:"" """, @"*" }, + { @"windows Tags:"" """, @"windows tokenizedPackageId:windows*" }, + { @"windows Tags:"" "" Tags:sdk", @"+tags:sdk windows" }, + + // Duplicate search terms on the same query field are folded + { "a a", "a tokenizedPackageId:a*^20" }, + { "title:a title:a", "title:a" }, + { "tag:a tags:a", "tags:a" }, + { "tags:a,a", "tags:a" }, + + // Single word query terms are unquoted. + { @"""a""", "a tokenizedPackageId:a*^20" }, + { @"title:""a""", "title:a" }, + + // Lucene keywords are removed unless quoted with other terms + { @"AND OR NOT", @"*" }, + { @"and or not", @"*" }, + { @"""AND""", @"*" }, + { @"""OR""", @"*" }, + { @"""NOT""", @"*" }, + { @"""AND"" ""OR"" ""NOT""", @"*" }, + { @"""AND OR NOT""", @"""AND OR NOT"" tokenizedPackageId:AND\ OR\ NOT*" }, + { @"""AND OR""", @"""AND OR"" tokenizedPackageId:AND\ OR*" }, + { @"""OR NOT""", @"""OR NOT"" tokenizedPackageId:OR\ NOT*" }, + { @"""AND NOT""", @"""AND NOT"" tokenizedPackageId:AND\ NOT*" }, + { @""" AND""", @""" AND"" tokenizedPackageId:AND*" }, + { @""" OR """, @""" OR "" tokenizedPackageId:OR*" }, + { @"""NOT """, @"""NOT "" tokenizedPackageId:NOT*" }, + { @"hello AND world", @"hello world (+hello +world)^2 tokenizedPackageId:world*" }, + { @"hello OR world", @"hello world (+hello +world)^2 tokenizedPackageId:world*" }, + { @"hello NOT world", @"hello world (+hello +world)^2 tokenizedPackageId:world*" }, + { @"title:""hello AND world""", @"title:""hello AND world""" }, + { @"title:""hello OR world""", @"title:""hello OR world""" }, + { @"title:""hello NOT world""", @"title:""hello NOT world""" }, + + // Special characters are escaped + { @"title:+ description:""+""", @"+title:\+ +description:\+" }, + { @"title:- description:""-""", @"+title:\- +description:\-" }, + { @"title:& description:""&""", @"+title:\& +description:\&" }, + { @"title:| description:""|""", @"+title:\| +description:\|" }, + { @"title:! description:""!""", @"+title:\! +description:\!" }, + { @"title:( description:""(""", @"+title:\( +description:\(" }, + { @"title:) description:"")""", @"+title:\) +description:\)" }, + { @"title:{ description:""{""", @"+title:\{ +description:\{" }, + { @"title:} description:""}""", @"+title:\} +description:\}" }, + { @"title:[ description:""[""", @"+title:\[ +description:\[" }, + { @"title:] description:""]""", @"+title:\] +description:\]" }, + { @"title:~ description:""~""", @"+title:\~ +description:\~" }, + { @"title:* description:""*""", @"+title:\* +description:\*" }, + { @"title:? description:""?""", @"+title:\? +description:\?" }, + { @"title:\ description:""\""", @"+title:\\ +description:\\" }, + { @"title:/ description:""/""", @"+title:\/ +description:\/" }, + { @"title:"":""", @"title:\:" }, + + { @"+ - & | ! ( ) { } [ ] ~ * ? \ / "":""", + @"\+ \- \& \| \! \( \) \{ \} \[ \] \~ \* \? \\ \/ \: " + + @"(+\+ +\- +\& +\| +\! +\( +\) +\{ +\} +\[ +\] +\~ +\* +\? +\\ +\/ +\:)^2 tokenizedPackageId:\:*^20"}, + + // Unicode surrogate pairs + { "A𠈓C", "A𠈓C tokenizedPackageId:A𠈓C*" }, + { "packageId:A𠈓C", "packageId:A𠈓C" }, + { @"""A𠈓C"" packageId:""A𠈓C""", "+packageId:A𠈓C A𠈓C" }, + { @"(𠈓) packageId:(𠈓)", @"+packageId:\(𠈓\) \(𠈓\)" }, + { "A𠈓C packageId:A𠈓C A𠈓C packageId:A𠈓C hello packageId:hello", + "+packageId:(A𠈓C hello) A𠈓C hello (+A𠈓C +hello)^2" }, + }; + + foreach (var datum in data) + { + yield return new object[] { datum.Key, datum.Value }; + } + } + + public static IEnumerable GenerateQueriesManyClauses() + { + object[] Setup(int nonFieldScopedTerms = 0, int fieldScopedTerms = 0, bool shouldThrow = false) + { + return new object[] { nonFieldScopedTerms, fieldScopedTerms, shouldThrow }; + } + + // There must be less than 1025 clauses. Each non-field-scoped term count as a clause. + yield return Setup(nonFieldScopedTerms: 1024, shouldThrow: false); + yield return Setup(nonFieldScopedTerms: 1025, shouldThrow: true); + + // Each field-scoped terms count as a clause each, and the field-scope itself counts as a clause if it has more than one term. + yield return Setup(fieldScopedTerms: 1023, shouldThrow: false); + yield return Setup(fieldScopedTerms: 1024, shouldThrow: true); + + yield return Setup(nonFieldScopedTerms: 1023, fieldScopedTerms: 1, shouldThrow: false); + yield return Setup(nonFieldScopedTerms: 1024, fieldScopedTerms: 1, shouldThrow: true); + + yield return Setup(nonFieldScopedTerms: 1021, fieldScopedTerms: 2, shouldThrow: false); + yield return Setup(nonFieldScopedTerms: 1022, fieldScopedTerms: 2, shouldThrow: true); + } + + protected string GenerateQuery(int nonFieldScopedTerms, int fieldScopedTerms) + { + List GenerateTerms(int count) + { + return Enumerable.Range(0, count).Select(i => i.ToString()).ToList(); + } + + var nonFieldScoped = GenerateTerms(nonFieldScopedTerms); + var result = string.Join(" ", nonFieldScoped); + + if (fieldScopedTerms == 0) + { + return result; + } + + if (nonFieldScopedTerms > 0) + { + result += " "; + } + + var fieldScoped = GenerateTerms(fieldScopedTerms); + result += $"packageId:{string.Join(" packageId:", fieldScoped)}"; + + return result; + } + + public static IEnumerable GenerateQueryWithTooBigTerm() + { + yield return new object[] { new string('a', 32 * 1024 + 1) }; + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/SearchService/SecretRefresherFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/SearchService/SecretRefresherFacts.cs new file mode 100644 index 000000000..6509a3b86 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/SearchService/SecretRefresherFacts.cs @@ -0,0 +1,157 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Services.AzureSearch.Wrappers; +using NuGet.Services.KeyVault; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.SearchService +{ + public class SecretRefresherFacts + { + public class RefreshContinuouslyAsync : Facts + { + public RefreshContinuouslyAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task CanBeCancelledImmediately() + { + TokenSource.Cancel(); + + await Target.RefreshContinuouslyAsync(TokenSource.Token); + + Factory.Verify(x => x.RefreshAsync(It.IsAny()), Times.Never); + } + + [Fact] + public async Task SetsLastRefreshOnSuccess() + { + CancelAfter(reloads: 1); + + var before = Target.LastRefresh; + await Task.Delay(1); + await Target.RefreshContinuouslyAsync(TokenSource.Token); + var after = DateTimeOffset.UtcNow; + + Assert.NotEqual(before, Target.LastRefresh); + Assert.InRange(Target.LastRefresh, before, after); + } + + [Fact] + public async Task ReloadsUntilCancelled() + { + CancelAfter(reloads: 5); + Config.SecretRefreshFrequency = TimeSpan.Zero; + + await Target.RefreshContinuouslyAsync(TokenSource.Token); + + Factory.Verify(x => x.RefreshAsync(TokenSource.Token), Times.Exactly(5)); + Factory.Verify(x => x.RefreshAsync(It.IsAny()), Times.Exactly(5)); + } + + [Fact] + public async Task UsesReloadFrequencyOnSuccess() + { + FailAndCancelAfter(2); + + var before = Target.LastRefresh; + await Target.RefreshContinuouslyAsync(TokenSource.Token); + var after = Target.LastRefresh; + + Assert.Equal(before, Target.LastRefresh); + Assert.Equal(after, Target.LastRefresh); + } + + [Fact] + public async Task UsesReloadFailureRetryFrequencyOnSuccess() + { + FailAndCancelAfter(2); + Config.SecretRefreshFrequency = TimeSpan.Zero; + Config.SecretRefreshFailureRetryFrequency = TimeSpan.FromMilliseconds(100); + + await Target.RefreshContinuouslyAsync(TokenSource.Token); + + Factory.Verify(x => x.RefreshAsync(It.IsAny()), Times.Exactly(2)); + SystemTime.Verify(x => x.Delay(Config.SecretRefreshFailureRetryFrequency, TokenSource.Token), Times.Once); + SystemTime.Verify(x => x.Delay(It.IsAny(), It.IsAny()), Times.Once); + } + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + Factory = new Mock(); + SystemTime = new Mock(); + Config = new SearchServiceConfiguration(); + Options = new Mock>(); + Logger = output.GetLogger(); + + TokenSource = new CancellationTokenSource(); + + // Default test behavior is to cancel after the first invocation otherwise it is very easy to loop + // forever, which is annoying for the person writing the tests. + CancelAfter(reloads: 1); + + Config.SecretRefreshFrequency = TimeSpan.FromMilliseconds(100); + Config.SecretRefreshFailureRetryFrequency = TimeSpan.FromMilliseconds(20); + Options.Setup(x => x.Value).Returns(() => Config); + + Target = new SecretRefresher( + Factory.Object, + SystemTime.Object, + Options.Object, + Logger); + } + + public Mock Factory { get; } + public Mock SystemTime { get; } + public SearchServiceConfiguration Config { get; } + public Mock> Options { get; } + public RecordingLogger Logger { get; } + public CancellationTokenSource TokenSource { get; } + public SecretRefresher Target { get; } + + protected void FailAndCancelAfter(int reloads) + { + int count = 0; + Factory + .Setup(x => x.RefreshAsync(It.IsAny())) + .Returns(() => + { + count++; + if (count >= reloads) + { + TokenSource.Cancel(); + } + + throw new InvalidOperationException("Please retry later."); + }); + } + + protected void CancelAfter(int reloads) + { + int count = 0; + Factory + .Setup(x => x.RefreshAsync(It.IsAny())) + .Returns(Task.CompletedTask) + .Callback(() => + { + count++; + if (count >= reloads) + { + TokenSource.Cancel(); + } + }); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Support/Data.cs b/tests/NuGet.Services.AzureSearch.Tests/Support/Data.cs new file mode 100644 index 000000000..b3f4bab16 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Support/Data.cs @@ -0,0 +1,108 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Microsoft.Extensions.Options; +using Moq; +using NuGet.Services.AzureSearch.AuxiliaryFiles; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Support +{ + public class Data : V3Data + { + public static readonly DateTimeOffset DocumentLastUpdated = new DateTimeOffset(2018, 12, 14, 9, 30, 0, TimeSpan.Zero); + + public static void SetDocumentLastUpdated(IUpdatedDocument document, ITestOutputHelper output) + { + var currentTimestamp = document.LastUpdatedDocument; + output.WriteLine( + $"The commited document has a generated {nameof(document.LastUpdatedDocument)} value of " + + $"{currentTimestamp:O}. Replacing this value with {DocumentLastUpdated:O}."); + document.LastUpdatedDocument = DocumentLastUpdated; + } + + public static string[] Versions => new[] + { + "1.0.0", + "2.0.0+git", + "3.0.0-alpha.1", + FullVersion + }; + + public static string[] Owners => new[] + { + "Microsoft", + "azure-sdk", + }; + + public const int TotalDownloadCount = 1001; + + public static readonly SearchFilters SearchFilters = SearchFilters.IncludePrereleaseAndSemVer2; + + public static readonly AzureSearchScoringConfiguration Config = new AzureSearchScoringConfiguration + { + DownloadScoreBoost = 2 + }; + + private static IOptionsSnapshot Options + { + get + { + var mock = new Mock>(); + var config = new AzureSearchJobConfiguration + { + GalleryBaseUrl = GalleryBaseUrl, + FlatContainerBaseUrl = FlatContainerBaseUrl, + FlatContainerContainerName = FlatContainerContainerName, + Scoring = new AzureSearchScoringConfiguration(), + }; + + mock.Setup(o => o.Value).Returns(config); + + return mock.Object; + } + } + + private static BaseDocumentBuilder BaseDocumentBuilder => new BaseDocumentBuilder(Options); + + public static SearchDocument.Full SearchDocument => new SearchDocumentBuilder(BaseDocumentBuilder).FullFromDb( + PackageId, + SearchFilters.IncludePrereleaseAndSemVer2, + Versions, + isLatestStable: false, + isLatest: true, + fullVersion: FullVersion, + package: PackageEntity, + owners: Owners, + totalDownloadCount: TotalDownloadCount, + isExcludedByDefault: false); + + public static HijackDocumentChanges HijackDocumentChanges => new HijackDocumentChanges( + delete: false, + updateMetadata: true, + latestStableSemVer1: false, + latestSemVer1: true, + latestStableSemVer2: false, + latestSemVer2: true); + + public static HijackDocument.Full HijackDocument => new HijackDocumentBuilder(BaseDocumentBuilder).FullFromDb( + PackageId, + HijackDocumentChanges, + PackageEntity); + + public static AuxiliaryFileMetadata GetAuxiliaryFileMetadata(string etag) => new AuxiliaryFileMetadata( + DateTimeOffset.MinValue, + TimeSpan.Zero, + fileSize: 0, + etag: etag); + + public static AuxiliaryFileResult GetAuxiliaryFileResult(T data, string etag) where T : class + { + return new AuxiliaryFileResult( + modified: true, + data: data, + metadata: GetAuxiliaryFileMetadata(etag)); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Support/SerializationUtilities.cs b/tests/NuGet.Services.AzureSearch.Tests/Support/SerializationUtilities.cs new file mode 100644 index 000000000..4af34a9ef --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Support/SerializationUtilities.cs @@ -0,0 +1,47 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Search; +using Microsoft.Azure.Search.Models; + +namespace NuGet.Services.AzureSearch.Support +{ + public class SerializationUtilities + { + public static async Task SerializeToJsonAsync(T obj) where T : class + { + using (var testHandler = new TestHttpClientHandler()) + using (var serviceClient = new SearchServiceClient( + "unit-test-service", + new SearchCredentials("unit-test-api-key"), + testHandler)) + { + var indexClient = serviceClient.Indexes.GetClient("unit-test-index"); + await indexClient.Documents.IndexAsync(IndexBatch.Upload(new[] { obj })); + return testHandler.LastRequestBody; + } + } + + private class TestHttpClientHandler : HttpClientHandler + { + public string LastRequestBody { get; private set; } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.Content != null) + { + LastRequestBody = await request.Content.ReadAsStringAsync(); + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(LastRequestBody ?? string.Empty), + }; + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/VersionList/FilteredVersionListFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/VersionList/FilteredVersionListFacts.cs new file mode 100644 index 000000000..aed40f324 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/VersionList/FilteredVersionListFacts.cs @@ -0,0 +1,958 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Versioning; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace NuGet.Services.AzureSearch +{ + public class FilteredVersionListFacts + { + public class Upsert : BaseFacts + { + [Fact] + public void ReplacesLatestFullVersionByNormalizedVersion() + { + ClearList(); + _list.Upsert(new FilteredVersionProperties("1.02.0-Alpha.1+git", listed: true)); + var properties = new FilteredVersionProperties("1.2.0.0-ALPHA.1+somethingelse", listed: true); + + var output = _list.Upsert(properties); + + Assert.Equal(SearchIndexChangeType.UpdateLatest, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(properties.ParsedVersion), + HijackIndexChange.SetLatestToTrue(properties.ParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(properties.FullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(properties.ParsedVersion, _list._latestOrNull); + Assert.Equal(properties.FullVersion, _list._latestOrNull.ToFullString()); + Assert.Equal(new[] { properties.FullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { properties.ParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void ReplacesNonLatestFullVersionByNormalizedVersion() + { + ClearList(); + var latest = new FilteredVersionProperties("2.0.0", listed: true); + _list.Upsert(latest); + _list.Upsert(new FilteredVersionProperties("1.02.0-Alpha.1+git", listed: true)); + var properties = new FilteredVersionProperties("1.2.0.0-ALPHA.1+somethingelse", listed: true); + + var output = _list.Upsert(properties); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(properties.ParsedVersion), + HijackIndexChange.SetLatestToFalse(properties.ParsedVersion), + HijackIndexChange.SetLatestToTrue(latest.ParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(latest.FullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(latest.ParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { properties.FullVersion, latest.FullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal( + new[] { properties.ParsedVersion, latest.ParsedVersion }, + _list._versions.Keys.ToArray()); + } + + [Theory] + [MemberData(nameof(PreviousAndNextVersions))] + public void AddingUnlistedVersion(string version) + { + var unlisted = new FilteredVersionProperties(version, listed: false); + + var output = _list.Upsert(unlisted); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(unlisted.ParsedVersion), + HijackIndexChange.SetLatestToFalse(unlisted.ParsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal( + new[] { unlisted.ParsedVersion, InitialParsedVersion }.OrderBy(x => x).ToArray(), + _list._versions.Keys.ToArray()); + } + + [Fact] + public void AddingVeryFirstVersion() + { + ClearList(); + + var output = _list.Upsert(_initialVersionProperties); + + Assert.Equal(SearchIndexChangeType.AddFirst, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(InitialParsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Theory] + [MemberData(nameof(PreviousAndNextVersions))] + public void AddingFirstListedVersion(string version) + { + StartWithUnlisted(); + var listed = new FilteredVersionProperties(version, listed: true); + + var output = _list.Upsert(listed); + + Assert.Equal(SearchIndexChangeType.AddFirst, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(listed.ParsedVersion), + HijackIndexChange.SetLatestToTrue(listed.ParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(listed.FullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(listed.ParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { listed.FullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal( + new[] { InitialParsedVersion, listed.ParsedVersion }.OrderBy(x => x).ToArray(), + _list._versions.Keys.ToArray()); + } + + [Fact] + public void UnlistingLatestAndNoOtherVersions() + { + var output = _list.Upsert(_unlistedVersionProperties); + + Assert.Equal(SearchIndexChangeType.Delete, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(InitialParsedVersion), + HijackIndexChange.SetLatestToFalse(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Null(_list.GetLatestVersionInfo()); + Assert.Null(_list._latestOrNull); + Assert.Equal(new[] { InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Theory] + [MemberData(nameof(PreviousAndNextVersions))] + public void UnlistingLatestWithOtherUnlistedVersions(string version) + { + var unlisted = new FilteredVersionProperties(version, listed: false); + _list.Upsert(unlisted); + + var output = _list.Upsert(_unlistedVersionProperties); + + Assert.Equal(SearchIndexChangeType.Delete, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(InitialParsedVersion), + HijackIndexChange.SetLatestToFalse(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Null(_list.GetLatestVersionInfo()); + Assert.Null(_list._latestOrNull); + Assert.Equal( + new[] { InitialParsedVersion, unlisted.ParsedVersion }.OrderBy(x => x).ToArray(), + _list._versions.Keys.ToArray()); + } + + [Fact] + public void UnlistingLatestWithOtherListedVersion() + { + var listed = new FilteredVersionProperties(PreviousFullVersion, listed: true); + _list.Upsert(listed); + + var output = _list.Upsert(_unlistedVersionProperties); + + Assert.Equal(SearchIndexChangeType.DowngradeLatest, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(InitialParsedVersion), + HijackIndexChange.SetLatestToFalse(InitialParsedVersion), + HijackIndexChange.SetLatestToTrue(PreviousParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(listed.FullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(listed.ParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { listed.FullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { listed.ParsedVersion, InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void UnlistingNonLatestVersion() + { + var listed = new FilteredVersionProperties(NextFullVersion, listed: true); + _list.Upsert(listed); + + var output = _list.Upsert(_unlistedVersionProperties); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(InitialParsedVersion), + HijackIndexChange.SetLatestToFalse(InitialParsedVersion), + HijackIndexChange.SetLatestToTrue(NextParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(listed.FullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(listed.ParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { listed.FullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { InitialParsedVersion, listed.ParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void AddingListedNonLatestVersion() + { + var listed = new FilteredVersionProperties(PreviousFullVersion, listed: true); + + var output = _list.Upsert(listed); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(listed.ParsedVersion), + HijackIndexChange.SetLatestToFalse(listed.ParsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { PreviousFullVersion, InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { listed.ParsedVersion, InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void RelistingNonLatestVersion() + { + var listed = new FilteredVersionProperties(PreviousFullVersion, listed: true); + _list.Upsert(listed); + + var output = _list.Upsert(listed); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(listed.ParsedVersion), + HijackIndexChange.SetLatestToFalse(listed.ParsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { listed.FullVersion, InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { listed.ParsedVersion, InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void UnlistingUnlistedVersionAndNoOtherVersions() + { + StartWithUnlisted(); + + var output = _list.Upsert(_unlistedVersionProperties); + + Assert.Equal(SearchIndexChangeType.Delete, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(InitialParsedVersion), + HijackIndexChange.SetLatestToFalse(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Null(_list.GetLatestVersionInfo()); + Assert.Null(_list._latestOrNull); + Assert.Equal(new[] { InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void UnlistingNewVersionAndNoOtherVersions() + { + ClearList(); + + var output = _list.Upsert(_unlistedVersionProperties); + + Assert.Equal(SearchIndexChangeType.Delete, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(InitialParsedVersion), + HijackIndexChange.SetLatestToFalse(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Null(_list.GetLatestVersionInfo()); + Assert.Null(_list._latestOrNull); + Assert.Equal(new[] { InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void RelistingLatestVersion() + { + var output = _list.Upsert(_initialVersionProperties); + + Assert.Equal(SearchIndexChangeType.UpdateLatest, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(InitialParsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void NewVersionIsIntroduced() + { + ClearList(); + var listed = new FilteredVersionProperties(PreviousFullVersion, listed: true); + _list.Upsert(listed); + + var output = _list.Upsert(_initialVersionProperties); + + Assert.Equal(SearchIndexChangeType.UpdateLatest, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(InitialParsedVersion), + HijackIndexChange.SetLatestToFalse(PreviousParsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { listed.FullVersion, InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { listed.ParsedVersion, InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + } + + public class Remove : BaseFacts + { + [Fact] + public void RemovesByNormalizedVersion() + { + ClearList(); + _list.Upsert(new FilteredVersionProperties("1.02.0-Alpha.1+git", listed: true)); + var removedVersion = NuGetVersion.Parse("1.02.0-Alpha.1+git"); + + var output = _list.Remove(removedVersion); + + Assert.Equal(SearchIndexChangeType.Delete, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(removedVersion), + HijackIndexChange.SetLatestToFalse(removedVersion), + }, + output.Hijack.ToArray()); + Assert.Null(_list.GetLatestVersionInfo()); + Assert.Null(_list._latestOrNull); + Assert.Empty(_list._versions.Keys); + } + + [Theory] + [MemberData(nameof(PreviousAndNextVersions))] + public void RemovesNewVersion(string version) + { + var parsedVersion = NuGetVersion.Parse(version); + + var output = _list.Remove(parsedVersion); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(parsedVersion), + HijackIndexChange.SetLatestToFalse(parsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Theory] + [MemberData(nameof(PreviousAndNextVersions))] + public void RemovesUnlistedVersion(string version) + { + var unlisted = new FilteredVersionProperties(version, listed: false); + _list.Upsert(unlisted); + + var output = _list.Remove(unlisted.ParsedVersion); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(unlisted.ParsedVersion), + HijackIndexChange.SetLatestToFalse(unlisted.ParsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void RemovesLatestVersion() + { + var latest = new FilteredVersionProperties(NextFullVersion, listed: true); + _list.Upsert(latest); + + var output = _list.Remove(latest.ParsedVersion); + + Assert.Equal(SearchIndexChangeType.DowngradeLatest, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(latest.ParsedVersion), + HijackIndexChange.SetLatestToFalse(latest.ParsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void RemovesVeryLastVersionWhenLastVersionIsListed() + { + StartWithUnlisted(); + + var output = _list.Remove(InitialParsedVersion); + + Assert.Equal(SearchIndexChangeType.Delete, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(InitialParsedVersion), + HijackIndexChange.SetLatestToFalse(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Null(_list.GetLatestVersionInfo()); + Assert.Null(_list._latestOrNull); + Assert.Empty(_list._versions); + } + + [Fact] + public void RemovesVeryLastVersionWhenLastVersionIsUnlisted() + { + var output = _list.Remove(InitialParsedVersion); + + Assert.Equal(SearchIndexChangeType.Delete, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(InitialParsedVersion), + HijackIndexChange.SetLatestToFalse(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Null(_list.GetLatestVersionInfo()); + Assert.Null(_list._latestOrNull); + Assert.Empty(_list._versions); + } + + [Fact] + public void RemovesFromEmptyList() + { + ClearList(); + + var output = _list.Remove(InitialParsedVersion); + + Assert.Equal(SearchIndexChangeType.Delete, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(InitialParsedVersion), + HijackIndexChange.SetLatestToFalse(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Null(_list.GetLatestVersionInfo()); + Assert.Null(_list._latestOrNull); + Assert.Empty(_list._versions.Keys); + } + + [Theory] + [MemberData(nameof(PreviousAndNextVersions))] + public void RemovesLastListedVersion(string version) + { + var unlisted = new FilteredVersionProperties(version, listed: false); + _list.Upsert(unlisted); + + var output = _list.Remove(InitialParsedVersion); + + Assert.Equal(SearchIndexChangeType.Delete, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(InitialParsedVersion), + HijackIndexChange.SetLatestToFalse(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Null(_list.GetLatestVersionInfo()); + Assert.Null(_list._latestOrNull); + Assert.Equal(new[] { unlisted.ParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void RemovesNonLatestListedVersionWithOneOtherVersion() + { + var nonLatest = new FilteredVersionProperties(PreviousFullVersion, listed: true); + _list.Upsert(nonLatest); + + var output = _list.Remove(nonLatest.ParsedVersion); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(nonLatest.ParsedVersion), + HijackIndexChange.SetLatestToFalse(nonLatest.ParsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void RemovesNonLatestListedVersionWithTwoOtherVersions() + { + _list.Upsert(new FilteredVersionProperties(PreviousFullVersion, listed: true)); + _list.Upsert(new FilteredVersionProperties(NextFullVersion, listed: true)); + + var output = _list.Remove(InitialParsedVersion); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(InitialParsedVersion), + HijackIndexChange.SetLatestToFalse(InitialParsedVersion), + HijackIndexChange.SetLatestToTrue(NextParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(NextFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(NextParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { PreviousFullVersion, NextFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { PreviousParsedVersion, NextParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void RemovesLatestListedVersionWithTwoOtherVersions() + { + _list.Upsert(new FilteredVersionProperties(PreviousFullVersion, listed: true)); + _list.Upsert(new FilteredVersionProperties(NextFullVersion, listed: true)); + + var output = _list.Remove(NextParsedVersion); + + Assert.Equal(SearchIndexChangeType.DowngradeLatest, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.UpdateMetadata(NextParsedVersion), + HijackIndexChange.SetLatestToFalse(NextParsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + Enumerable.ToArray(output.Hijack)); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { PreviousFullVersion, InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { PreviousParsedVersion, InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + } + + public class Delete : BaseFacts + { + [Fact] + public void DeletesByNormalizedVersion() + { + ClearList(); + _list.Upsert(new FilteredVersionProperties("1.02.0-Alpha.1+git", listed: true)); + var deletedVersion = NuGetVersion.Parse("1.02.0-Alpha.1+git"); + + var output = _list.Delete(deletedVersion); + + Assert.Equal(SearchIndexChangeType.Delete, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.Delete(deletedVersion), + }, + output.Hijack.ToArray()); + Assert.Null(_list.GetLatestVersionInfo()); + Assert.Null(_list._latestOrNull); + Assert.Empty(_list._versions.Keys); + } + + [Theory] + [MemberData(nameof(PreviousAndNextVersions))] + public void DeletesNewVersion(string version) + { + var parsedVersion = NuGetVersion.Parse(version); + + var output = _list.Delete(parsedVersion); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.Delete(parsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Theory] + [MemberData(nameof(PreviousAndNextVersions))] + public void DeletesUnlistedVersion(string version) + { + var unlisted = new FilteredVersionProperties(version, listed: false); + _list.Upsert(unlisted); + + var output = _list.Delete(unlisted.ParsedVersion); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.Delete(unlisted.ParsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void DeletesLatestVersion() + { + var latest = new FilteredVersionProperties(NextFullVersion, listed: true); + _list.Upsert(latest); + + var output = _list.Delete(latest.ParsedVersion); + + Assert.Equal(SearchIndexChangeType.DowngradeLatest, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.Delete(latest.ParsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void DeletesVeryLastVersionWhenLastVersionIsListed() + { + StartWithUnlisted(); + + var output = _list.Delete(InitialParsedVersion); + + Assert.Equal(SearchIndexChangeType.Delete, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.Delete(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Null(_list.GetLatestVersionInfo()); + Assert.Null(_list._latestOrNull); + Assert.Empty(_list._versions); + } + + [Fact] + public void DeletesVeryLastVersionWhenLastVersionIsUnlisted() + { + var output = _list.Delete(InitialParsedVersion); + + Assert.Equal(SearchIndexChangeType.Delete, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.Delete(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Null(_list.GetLatestVersionInfo()); + Assert.Null(_list._latestOrNull); + Assert.Empty(_list._versions); + } + + /// + /// This behavior is important so that we can "reflow" deleted packages. Suppose there is a bug when all + /// versions of a package are deleted but there is still a search document left over. We should be able + /// to reflow a delete and force the document to be deleted. + /// + [Fact] + public void DeletesFromEmptyList() + { + ClearList(); + + var output = _list.Delete(InitialParsedVersion); + + Assert.Equal(SearchIndexChangeType.Delete, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.Delete(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Null(_list.GetLatestVersionInfo()); + Assert.Null(_list._latestOrNull); + Assert.Empty(_list._versions.Keys); + } + + [Theory] + [MemberData(nameof(PreviousAndNextVersions))] + public void DeletesLastListedVersion(string version) + { + var unlisted = new FilteredVersionProperties(version, listed: false); + _list.Upsert(unlisted); + + var output = _list.Delete(InitialParsedVersion); + + Assert.Equal(SearchIndexChangeType.Delete, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.Delete(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Null(_list.GetLatestVersionInfo()); + Assert.Null(_list._latestOrNull); + Assert.Equal(new[] { unlisted.ParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void DeletesNonLatestListedVersionWithOneOtherVersion() + { + var nonLatest = new FilteredVersionProperties(PreviousFullVersion, listed: true); + _list.Upsert(nonLatest); + + var output = _list.Delete(nonLatest.ParsedVersion); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.Delete(nonLatest.ParsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void DeletesNonLatestListedVersionWithTwoOtherVersions() + { + _list.Upsert(new FilteredVersionProperties(PreviousFullVersion, listed: true)); + _list.Upsert(new FilteredVersionProperties(NextFullVersion, listed: true)); + + var output = _list.Delete(InitialParsedVersion); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.Delete(InitialParsedVersion), + HijackIndexChange.SetLatestToTrue(NextParsedVersion), + }, + output.Hijack.ToArray()); + Assert.Equal(NextFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(NextParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { PreviousFullVersion, NextFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { PreviousParsedVersion, NextParsedVersion }, _list._versions.Keys.ToArray()); + } + + [Fact] + public void DeletesLatestListedVersionWithTwoOtherVersions() + { + _list.Upsert(new FilteredVersionProperties(PreviousFullVersion, listed: true)); + _list.Upsert(new FilteredVersionProperties(NextFullVersion, listed: true)); + + var output = _list.Delete(NextParsedVersion); + + Assert.Equal(SearchIndexChangeType.DowngradeLatest, output.Search); + Assert.Equal( + new[] + { + HijackIndexChange.Delete(NextParsedVersion), + HijackIndexChange.SetLatestToTrue(InitialParsedVersion), + }, + Enumerable.ToArray(output.Hijack)); + Assert.Equal(InitialFullVersion, _list.GetLatestVersionInfo().FullVersion); + Assert.Equal(InitialParsedVersion, _list._latestOrNull); + Assert.Equal(new[] { PreviousFullVersion, InitialFullVersion }, _list.GetLatestVersionInfo().ListedFullVersions); + Assert.Equal(new[] { PreviousParsedVersion, InitialParsedVersion }, _list._versions.Keys.ToArray()); + } + } + + public class LatestOrNull + { + [Fact] + public void UnlistedIsNotLatest() + { + var versions = new[] + { + new FilteredVersionProperties("1.0.0", listed: true), + new FilteredVersionProperties("1.0.1", listed: false), + }; + + var list = new FilteredVersionList(versions); + + Assert.Equal("1.0.0", list.GetLatestVersionInfo().FullVersion); + } + + [Fact] + public void LatestIsNullForEmpty() + { + var versions = new FilteredVersionProperties[0]; + + var list = new FilteredVersionList(versions); + + Assert.Null(list._latestOrNull); + } + + [Fact] + public void LatestIsNullForOnlyUnlisted() + { + var versions = new[] + { + new FilteredVersionProperties("1.0.0", listed: false), + new FilteredVersionProperties("1.0.1", listed: false), + }; + + var list = new FilteredVersionList(versions); + + Assert.Null(list._latestOrNull); + } + } + + public class FullVersions + { + [Fact] + public void ExcludesListedVersions() + { + var versions = new[] + { + new FilteredVersionProperties("1.0.0", listed: true), + new FilteredVersionProperties("1.0.1", listed: false), + }; + + var list = new FilteredVersionList(versions); + + Assert.Equal(new[] { "1.0.0" }, list.GetLatestVersionInfo().ListedFullVersions); + } + + [Fact] + public void OrdersBySemVer() + { + var versions = new[] + { + new FilteredVersionProperties("10.0.0", listed: true), + new FilteredVersionProperties("10.0.0-alpha", listed: true), + new FilteredVersionProperties("10.0.0-beta.10", listed: true), + new FilteredVersionProperties("10.0.0-beta.2", listed: true), + new FilteredVersionProperties("10.0.1", listed: true), + new FilteredVersionProperties("2.0.0", listed: true), + }; + + var list = new FilteredVersionList(versions); + + Assert.Equal( + new[] { "2.0.0", "10.0.0-alpha", "10.0.0-beta.2", "10.0.0-beta.10", "10.0.0", "10.0.1" }, + list.GetLatestVersionInfo().ListedFullVersions); + } + } + + public abstract class BaseFacts + { + protected const string PreviousFullVersion = "0.9.0"; + protected const string InitialFullVersion = "1.0.0"; + protected const string NextFullVersion = "1.1.0"; + protected static readonly NuGetVersion PreviousParsedVersion = NuGetVersion.Parse(PreviousFullVersion); + protected static readonly NuGetVersion InitialParsedVersion = NuGetVersion.Parse(InitialFullVersion); + protected static readonly NuGetVersion NextParsedVersion = NuGetVersion.Parse(NextFullVersion); + internal readonly FilteredVersionProperties _initialVersionProperties; + internal readonly FilteredVersionProperties _unlistedVersionProperties; + internal readonly FilteredVersionList _list; + + protected BaseFacts() + { + _initialVersionProperties = new FilteredVersionProperties(InitialFullVersion, listed: true); + _unlistedVersionProperties = new FilteredVersionProperties(InitialFullVersion, listed: false); + + var versions = new[] { _initialVersionProperties }; + + _list = new FilteredVersionList(versions); + } + + protected void ClearList() + { + foreach (var version in _list._versions.ToList()) + { + _list.Delete(version.Value.ParsedVersion); + } + } + + protected void StartWithUnlisted() + { + ClearList(); + _list.Upsert(_unlistedVersionProperties); + } + + public static IEnumerable PreviousAndNextVersions => new[] + { + new object[] { PreviousFullVersion }, + new object[] { NextFullVersion }, + }; + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/VersionList/MutableHijackDocumentChangesFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/VersionList/MutableHijackDocumentChangesFacts.cs new file mode 100644 index 000000000..168377241 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/VersionList/MutableHijackDocumentChangesFacts.cs @@ -0,0 +1,174 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using Xunit; + +namespace NuGet.Services.AzureSearch +{ + public class MutableHijackDocumentChangesFacts + { + public class Solidify + { + [Fact] + public void DefaultsNullToFalse() + { + var output = new MutableHijackDocumentChanges().Solidify(); + + Assert.False(output.Delete); + Assert.False(output.UpdateMetadata); + Assert.False(output.LatestStableSemVer1); + Assert.False(output.LatestSemVer1); + Assert.False(output.LatestStableSemVer2); + Assert.False(output.LatestSemVer2); + } + + [Fact] + public void MapsDeleteDocument() + { + var output = new MutableHijackDocumentChanges( + delete: true, + updateMetadata: false, + latestStableSemVer1: null, + latestSemVer1: null, + latestStableSemVer2: null, + latestSemVer2: null).Solidify(); + + Assert.True(output.Delete); + Assert.False(output.UpdateMetadata); + Assert.False(output.LatestStableSemVer1); + Assert.False(output.LatestSemVer1); + Assert.False(output.LatestStableSemVer2); + Assert.False(output.LatestSemVer2); + } + + [Fact] + public void MapsUpdateLatestDocument() + { + var output = new MutableHijackDocumentChanges( + delete: false, + updateMetadata: true, + latestStableSemVer1: true, + latestSemVer1: true, + latestStableSemVer2: true, + latestSemVer2: true).Solidify(); + + Assert.False(output.Delete); + Assert.True(output.UpdateMetadata); + Assert.True(output.LatestStableSemVer1); + Assert.True(output.LatestSemVer1); + Assert.True(output.LatestStableSemVer2); + Assert.True(output.LatestSemVer2); + } + } + + public class ApplyChange + { + [Fact] + public void DoesNotAllowUpdateMetadataToDeleteTransition() + { + var doc = new MutableHijackDocumentChanges(); + doc.ApplyChange(SearchFilters.Default, HijackIndexChangeType.UpdateMetadata); + + var ex = Assert.Throws( + () => doc.ApplyChange(SearchFilters.Default, HijackIndexChangeType.Delete)); + Assert.Equal( + "The hijack document has already been set to update metadata.", + ex.Message); + } + + [Fact] + public void DoesNotAllowDeleteToUpdateMetadataTransition() + { + var doc = new MutableHijackDocumentChanges(); + doc.ApplyChange(SearchFilters.Default, HijackIndexChangeType.Delete); + + var ex = Assert.Throws( + () => doc.ApplyChange(SearchFilters.Default, HijackIndexChangeType.UpdateMetadata)); + Assert.Equal( + "The hijack document has already been set to delete so metadata can't be updated.", + ex.Message); + } + + [Theory] + [MemberData(nameof(ChangeAndLatest))] + public void SetLatestWithDefaultSearchFilters(HijackIndexChangeType change, bool latest) + { + var doc = new MutableHijackDocumentChanges(); + + doc.ApplyChange(SearchFilters.Default, change); + + Assert.Equal(latest, doc.LatestStableSemVer1); + Assert.Null(doc.LatestSemVer1); + Assert.Null(doc.LatestStableSemVer2); + Assert.Null(doc.LatestSemVer2); + } + + [Theory] + [MemberData(nameof(ChangeAndLatest))] + public void SetLatestWithIncludePrereleaseSearchFilters(HijackIndexChangeType change, bool latest) + { + var doc = new MutableHijackDocumentChanges(); + + doc.ApplyChange(SearchFilters.IncludePrerelease, change); + + Assert.Null(doc.LatestStableSemVer1); + Assert.Equal(latest, doc.LatestSemVer1); + Assert.Null(doc.LatestStableSemVer2); + Assert.Null(doc.LatestSemVer2); + } + + [Theory] + [MemberData(nameof(ChangeAndLatest))] + public void SetLatestWithIncludeSemVer2SearchFilters(HijackIndexChangeType change, bool latest) + { + var doc = new MutableHijackDocumentChanges(); + + doc.ApplyChange(SearchFilters.IncludeSemVer2, change); + + Assert.Null(doc.LatestStableSemVer1); + Assert.Null(doc.LatestSemVer1); + Assert.Equal(latest, doc.LatestStableSemVer2); + Assert.Null(doc.LatestSemVer2); + } + + [Theory] + [MemberData(nameof(ChangeAndLatest))] + public void SetLatestWithIncludePrereleaseAndSemVer2SearchFilters(HijackIndexChangeType change, bool latest) + { + var doc = new MutableHijackDocumentChanges(); + + doc.ApplyChange(SearchFilters.IncludePrereleaseAndSemVer2, change); + + Assert.Null(doc.LatestStableSemVer1); + Assert.Null(doc.LatestSemVer1); + Assert.Null(doc.LatestStableSemVer2); + Assert.Equal(latest, doc.LatestSemVer2); + } + + public static IEnumerable ChangeAndLatest => new[] + { + new object[] { HijackIndexChangeType.SetLatestToTrue, true }, + new object[] { HijackIndexChangeType.SetLatestToFalse, false }, + }; + + [Fact] + public void DeleteTransitionClearsLatest() + { + var doc = new MutableHijackDocumentChanges(); + doc.ApplyChange(SearchFilters.Default, HijackIndexChangeType.SetLatestToTrue); + doc.ApplyChange(SearchFilters.IncludePrerelease, HijackIndexChangeType.SetLatestToTrue); + doc.ApplyChange(SearchFilters.IncludeSemVer2, HijackIndexChangeType.SetLatestToTrue); + doc.ApplyChange(SearchFilters.IncludePrereleaseAndSemVer2, HijackIndexChangeType.SetLatestToTrue); + + doc.ApplyChange(SearchFilters.Default, HijackIndexChangeType.Delete); + + Assert.Null(doc.LatestStableSemVer1); + Assert.Null(doc.LatestSemVer1); + Assert.Null(doc.LatestStableSemVer2); + Assert.Null(doc.LatestSemVer2); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/VersionList/MutableIndexChangesFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/VersionList/MutableIndexChangesFacts.cs new file mode 100644 index 000000000..9a3891403 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/VersionList/MutableIndexChangesFacts.cs @@ -0,0 +1,46 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Linq; +using NuGet.Versioning; +using Xunit; + +namespace NuGet.Services.AzureSearch +{ + public class MutableIndexChangesFacts + { + public class Solidify + { + [Fact] + public void MapsProperties() + { + var v1 = NuGetVersion.Parse("1.0.0"); + var changes = MutableIndexChanges.FromLatestIndexChanges(new Dictionary + { + { + SearchFilters.Default, + new LatestIndexChanges( + SearchIndexChangeType.AddFirst, + new List + { + HijackIndexChange.UpdateMetadata(v1), + HijackIndexChange.SetLatestToTrue(v1), + }) + }, + }); + + var solid = changes.Solidify(); + + Assert.Equal(new[] { SearchFilters.Default }, solid.Search.Keys.ToArray()); + Assert.Equal(SearchIndexChangeType.AddFirst, solid.Search[SearchFilters.Default]); + Assert.False(solid.Hijack[v1].Delete); + Assert.True(solid.Hijack[v1].UpdateMetadata); + Assert.True(solid.Hijack[v1].LatestStableSemVer1); + Assert.False(solid.Hijack[v1].LatestSemVer1); + Assert.False(solid.Hijack[v1].LatestStableSemVer2); + Assert.False(solid.Hijack[v1].LatestSemVer2); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/VersionList/TestExtensionMethods.cs b/tests/NuGet.Services.AzureSearch.Tests/VersionList/TestExtensionMethods.cs new file mode 100644 index 000000000..c23e54c95 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/VersionList/TestExtensionMethods.cs @@ -0,0 +1,20 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using NuGet.Versioning; + +namespace NuGet.Services.AzureSearch +{ + internal static class TestExtensionMethods + { + public static MutableIndexChanges Upsert(this VersionLists versionList, VersionProperties version) + { + return versionList.Upsert(version.FullVersion, version.Data); + } + + public static MutableIndexChanges Delete(this VersionLists versionList, string fullOrOriginalVersion) + { + return versionList.Delete(NuGetVersion.Parse(fullOrOriginalVersion)); + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/VersionList/VersionListDataClientFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/VersionList/VersionListDataClientFacts.cs new file mode 100644 index 000000000..89e3b8f69 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/VersionList/VersionListDataClientFacts.cs @@ -0,0 +1,304 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Moq; +using NuGetGallery; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch +{ + public class VersionListDataClientFacts + { + public class TheReadAsyncMethod : Facts + { + public TheReadAsyncMethod(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task ReturnsEmptyListWhenFileDoesNotExist() + { + var output = await _target.ReadAsync(_id); + + Assert.NotNull(output); + Assert.NotNull(output.Result); + Assert.NotNull(output.AccessCondition); + Assert.NotNull(output.Result.VersionProperties); + Assert.Empty(output.Result.VersionProperties); + Assert.Equal("*", output.AccessCondition.IfNoneMatchETag); + Assert.Null(output.AccessCondition.IfMatchETag); + } + + [Theory] + [MemberData(nameof(ExpectedPaths))] + public async Task UsesExpectedContainerAndFileName(string path, string expected) + { + _config.StoragePath = path; + + await _target.ReadAsync(_id); + + _cloudBlobClient.Verify( + x => x.GetContainerReference(_config.StorageContainer), + Times.Once); + _cloudBlobContainer.Verify( + x => x.GetBlobReference(expected), + Times.Once); + } + + [Fact] + public async Task DeserializesJson() + { + var version2 = "2.0.0-beta.2"; + var version10 = "10.0.0"; + var versionList = Encoding.UTF8.GetBytes(@"{ + ""VersionProperties"": { + """ + version10 + @""": {}, + """ + version2 + @""": { + ""Listed"": true, + ""SemVer2"": true + } + } +}"); + var etag = "\"some-etag\""; + _cloudBlob + .Setup(x => x.ETag) + .Returns(etag); + _cloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ReturnsAsync(() => new MemoryStream(versionList)); + + var output = await _target.ReadAsync(_id); + + Assert.NotNull(output); + Assert.NotNull(output.Result); + Assert.NotNull(output.AccessCondition); + Assert.NotNull(output.Result.VersionProperties); + Assert.Equal(etag, output.AccessCondition.IfMatchETag); + Assert.Null(output.AccessCondition.IfNoneMatchETag); + var versions = output.Result.VersionProperties; + Assert.Equal(2, versions.Count); + Assert.Equal(new[] { version10, version2 }, versions.Keys.OrderBy(x => x).ToArray()); + Assert.True(versions[version2].Listed); + Assert.True(versions[version2].SemVer2); + Assert.False(versions[version10].Listed); + Assert.False(versions[version10].SemVer2); + } + } + + public class TheTryReplaceAsyncMethod : Facts + { + public TheTryReplaceAsyncMethod(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task ThrowsForOtherStorageExceptions() + { + var expected = new StorageException(new RequestResult { HttpStatusCode = (int)HttpStatusCode.InternalServerError }, "Fail.", null); + _cloudBlob + .Setup(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(expected); + + var actual = await Assert.ThrowsAsync(() => _target.TryReplaceAsync(_id, _versionList, _accessCondition.Object)); + + Assert.Same(expected, actual); + } + + [Fact] + public async Task ReturnsFalseForPreconditionFailed() + { + _cloudBlob + .Setup(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new StorageException(new RequestResult { HttpStatusCode = (int)HttpStatusCode.PreconditionFailed }, "Fail.", null)); + + var success = await _target.TryReplaceAsync(_id, _versionList, _accessCondition.Object); + + Assert.False(success); + } + + [Theory] + [MemberData(nameof(ExpectedPaths))] + public async Task UsesExpectedStorageParameters(string path, string expected) + { + _config.StoragePath = path; + + var success = await _target.TryReplaceAsync(_id, _versionList, _accessCondition.Object); + + Assert.True(success); + _cloudBlobClient.Verify( + x => x.GetContainerReference(_config.StorageContainer), + Times.Once); + _cloudBlobContainer.Verify( + x => x.GetBlobReference(expected), + Times.Once); + _cloudBlob.Verify( + x => x.UploadFromStreamAsync( + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task SerializesWithoutBOM() + { + await _target.TryReplaceAsync(_id, _versionList, _accessCondition.Object); + + var bytes = Assert.Single(_savedBytes); + Assert.Equal((byte)'{', bytes[0]); + } + + [Fact] + public async Task SetsContentType() + { + await _target.TryReplaceAsync(_id, _versionList, _accessCondition.Object); + + Assert.Equal("application/json", _cloudBlob.Object.Properties.ContentType); + } + + [Fact] + public async Task SerializesWithIndentation() + { + await _target.TryReplaceAsync(_id, _versionList, _accessCondition.Object); + + var json = Assert.Single(_savedStrings); + Assert.Contains("\n", json); + } + + [Fact] + public async Task SerializesVersionsInSemVerOrder() + { + var versionList = new VersionListData(new Dictionary + { + { "2.0.0", new VersionPropertiesData(listed: true, semVer2: false) }, + { "1.0.0-beta.2", new VersionPropertiesData(listed: true, semVer2: true) }, + { "10.0.0", new VersionPropertiesData(listed: false, semVer2: false) }, + { "1.0.0-beta.10", new VersionPropertiesData(listed: true, semVer2: true) }, + }); + + await _target.TryReplaceAsync(_id, versionList, _accessCondition.Object); + + var json = Assert.Single(_savedStrings); + Assert.Equal(@"{ + ""VersionProperties"": { + ""1.0.0-beta.2"": { + ""Listed"": true, + ""SemVer2"": true + }, + ""1.0.0-beta.10"": { + ""Listed"": true, + ""SemVer2"": true + }, + ""2.0.0"": { + ""Listed"": true + }, + ""10.0.0"": {} + } +}", json); + } + } + + public abstract class Facts + { + protected readonly string _id; + protected readonly Mock _accessCondition; + protected readonly VersionListData _versionList; + protected readonly Mock _cloudBlobClient; + protected readonly Mock _cloudBlobContainer; + protected readonly Mock _cloudBlob; + protected readonly Mock> _options; + protected readonly RecordingLogger _logger; + protected readonly AzureSearchJobConfiguration _config; + protected readonly VersionListDataClient _target; + + protected readonly List _savedBytes = new List(); + protected readonly List _savedStrings = new List(); + + public static IEnumerable ExpectedPaths => new[] + { + new object[] { "/", "version-lists/nuget.versioning.json" }, + new object[] { "", "version-lists/nuget.versioning.json" }, + new object[] { null, "version-lists/nuget.versioning.json" }, + new object[] { "/search/", "search/version-lists/nuget.versioning.json" }, + new object[] { "/search", "search/version-lists/nuget.versioning.json" }, + new object[] { "search/", "search/version-lists/nuget.versioning.json" }, + new object[] { "search", "search/version-lists/nuget.versioning.json" }, + new object[] { "/search/1/", "search/1/version-lists/nuget.versioning.json" }, + new object[] { "/search/1", "search/1/version-lists/nuget.versioning.json" }, + new object[] { "search/1", "search/1/version-lists/nuget.versioning.json" }, + new object[] { "search/1/", "search/1/version-lists/nuget.versioning.json" }, + }; + + public Facts(ITestOutputHelper output) + { + _id = "NuGet.Versioning"; + _accessCondition = new Mock(); + _versionList = new VersionListData(new Dictionary + { + { "2.0.0", new VersionPropertiesData(listed: true, semVer2: false) }, + }); + + _cloudBlobClient = new Mock(); + _cloudBlobContainer = new Mock(); + _cloudBlob = new Mock(); + _options = new Mock>(); + _logger = output.GetLogger(); + _config = new AzureSearchJobConfiguration + { + StorageContainer = "unit-test-container", + }; + + _options + .Setup(x => x.Value) + .Returns(() => _config); + _cloudBlobClient + .Setup(x => x.GetContainerReference(It.IsAny())) + .Returns(() => _cloudBlobContainer.Object); + _cloudBlobContainer + .Setup(x => x.GetBlobReference(It.IsAny())) + .Returns(() => _cloudBlob.Object); + _cloudBlob + .Setup(x => x.UploadFromStreamAsync(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask) + .Callback((s, _) => + { + using (s) + using (var buffer = new MemoryStream()) + { + s.CopyTo(buffer); + var bytes = buffer.ToArray(); + _savedBytes.Add(bytes); + _savedStrings.Add(Encoding.UTF8.GetString(bytes)); + } + }); + _cloudBlob + .Setup(x => x.OpenReadAsync(It.IsAny())) + .ThrowsAsync(new StorageException( + new RequestResult + { + HttpStatusCode = (int)HttpStatusCode.NotFound, + }, + message: "Not found.", + inner: null)); + _cloudBlob + .Setup(x => x.Properties) + .Returns(new CloudBlockBlob(new Uri("https://example/blob")).Properties); + + _target = new VersionListDataClient( + _cloudBlobClient.Object, + _options.Object, + _logger); + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/VersionList/VersionListsFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/VersionList/VersionListsFacts.cs new file mode 100644 index 000000000..26f920d39 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/VersionList/VersionListsFacts.cs @@ -0,0 +1,1690 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using NuGet.Services.V3.Support; +using NuGet.Versioning; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch +{ + public class VersionListsFacts + { + public class Constructor : BaseFacts + { + [Fact] + public void CategorizesVersionsByFilterPredicate() + { + var list = Create( + _stableSemVer1Listed, + _prereleaseSemVer1Listed, + _stableSemVer2Listed, + _prereleaseSemVer2Listed); + + Assert.Equal( + new[] + { + SearchFilters.Default, + SearchFilters.IncludePrerelease, + SearchFilters.IncludeSemVer2, + SearchFilters.IncludePrereleaseAndSemVer2, + }, + list._versionLists.Keys); + Assert.Equal( + new[] { StableSemVer1 }, + list._versionLists[SearchFilters.Default].GetLatestVersionInfo().ListedFullVersions); + Assert.Equal( + new[] { StableSemVer1, PrereleaseSemVer1 }, + list._versionLists[SearchFilters.IncludePrerelease].GetLatestVersionInfo().ListedFullVersions); + Assert.Equal( + new[] { StableSemVer1, StableSemVer2 }, + list._versionLists[SearchFilters.IncludeSemVer2].GetLatestVersionInfo().ListedFullVersions); + Assert.Equal( + new[] { StableSemVer1, PrereleaseSemVer1, StableSemVer2, PrereleaseSemVer2 }, + list._versionLists[SearchFilters.IncludePrereleaseAndSemVer2].GetLatestVersionInfo().ListedFullVersions); + } + + [Fact] + public void AllowsAllEmptyLists() + { + var list = Create(); + + Assert.Equal( + new[] + { + SearchFilters.Default, + SearchFilters.IncludePrerelease, + SearchFilters.IncludeSemVer2, + SearchFilters.IncludePrereleaseAndSemVer2, + }, + list._versionLists.Keys); + Assert.Empty(list._versionLists[SearchFilters.Default]._versions); + Assert.Empty(list._versionLists[SearchFilters.IncludePrerelease]._versions); + Assert.Empty(list._versionLists[SearchFilters.IncludeSemVer2]._versions); + Assert.Empty(list._versionLists[SearchFilters.IncludePrereleaseAndSemVer2]._versions); + } + + [Fact] + public void AllowsSomeEmptyLists() + { + var list = Create(_prereleaseSemVer2Listed); + + Assert.Equal( + new[] + { + SearchFilters.Default, + SearchFilters.IncludePrerelease, + SearchFilters.IncludeSemVer2, + SearchFilters.IncludePrereleaseAndSemVer2, + }, + list._versionLists.Keys); + Assert.Empty(list._versionLists[SearchFilters.Default]._versions); + Assert.Empty(list._versionLists[SearchFilters.IncludePrerelease]._versions); + Assert.Empty(list._versionLists[SearchFilters.IncludeSemVer2]._versions); + Assert.Equal( + new[] { PrereleaseSemVer2 }, + list._versionLists[SearchFilters.IncludePrereleaseAndSemVer2].GetLatestVersionInfo().ListedFullVersions); + } + } + + public class GetVersionListData : BaseFacts + { + [Fact] + public void ReturnsCurrentSetOfVersions() + { + var list = new VersionLists(new VersionListData(new Dictionary())); + + // add + list.Upsert(_stableSemVer1Listed); + + // delete + list.Upsert(_stableSemVer2Listed); + list.Delete(_stableSemVer2Listed.FullVersion); + + // delete with different case + list.Upsert(_prereleaseSemVer1Listed); + list.Delete(_prereleaseSemVer1Listed.FullVersion.ToUpper()); + + // unlist with different case + list.Upsert(_prereleaseSemVer2Listed); + list.Upsert(new VersionProperties( + _prereleaseSemVer2Listed.FullVersion.ToUpper(), + new VersionPropertiesData(listed: false, semVer2: true))); + + var data = list.GetVersionListData(); + Assert.Equal( + new[] { StableSemVer1, PrereleaseSemVer2.ToUpper() }, + data.VersionProperties.Keys.ToArray()); + } + } + + /// + /// This test suite produces every possible initial version state and version list change set to make sure no + /// exceptions are thrown. This has value given the number of calls in + /// the implementation. + /// + public class FullEnumerationOfPossibleTestCases + { + private readonly ITestOutputHelper _output; + + public FullEnumerationOfPossibleTestCases(ITestOutputHelper output) + { + _output = output ?? throw new ArgumentNullException(nameof(output)); + } + + [Fact] + public async Task AllTestCasesPass() + { + // Arrange + // 4 versions are used since it initially did not find any more bugs than 5 versions and, on most + // machines will run in under 2 seconds. + var versions = new[] { "1.0.0", "2.0.0", "3.0.0", "4.0.0" }; + var testCases = new ConcurrentBag(EnumerateTestCases(versions)); + _output.WriteLine($"Running {testCases.Count} test cases."); + var testResults = new ConcurrentBag(); + + // Run the test cases in parallel to improve test duration. + var tasks = Enumerable + .Range(0, 8) + .Select(i => Task.Run(() => + { + while (testCases.TryTake(out var testCase)) + { + var testResult = ExecuteTestCase(testCase); + testResults.Add(testResult); + } + })); + + // Act + await Task.WhenAll(tasks); + + // Assert + _output.WriteLine("Analyzing results."); + const int maxTestFailuresToOutput = 5; + var failureCount = 0; + foreach (var testResult in testResults) + { + if (testResult.Exception == null) + { + continue; + } + + failureCount++; + + if (failureCount > maxTestFailuresToOutput) + { + if (failureCount == maxTestFailuresToOutput + 1) + { + _output.WriteLine($"{maxTestFailuresToOutput} test failures have been shown. The rest will be hidden."); + } + + continue; + } + + _output.WriteLine("========== TEST CASE FAILURE =========="); + _output.WriteLine($"Initial state ({testResult.TestCase.InitialState.Length} versions):"); + foreach (var version in testResult.TestCase.InitialState) + { + _output.WriteLine( + $" - {version.FullVersion} ({(version.Data.Listed ? "listed" : "unlisted")})"); + } + _output.WriteLine($"Applied changes ({testResult.TestCase.ChangesToApply.Count} changes):"); + foreach (var version in testResult.TestCase.ChangesToApply) + { + _output.WriteLine( + $" - {(version.IsDelete ? "Delete" : "Upsert")} {version.FullVersion}" + + (version.IsDelete ? string.Empty : (version.Data.Listed ? " as listed" : " as unlisted"))); + } + _output.WriteLine("Exception:"); + _output.WriteLine(testResult.Exception.ToString()); + _output.WriteLine("======================================="); + _output.WriteLine(string.Empty); + } + + _output.WriteLine($"There were {failureCount} failed test cases."); + Assert.Equal(0, failureCount); + } + + private static TestResult ExecuteTestCase(TestCase testCase) + { + // Arrange + var list = ApplyChangesInternal.Create(testCase.InitialState); + + try + { + // Act & Assert + var output = list.ApplyChangesInternal(testCase.ChangesToApply); + + // In a simplistic way, determine the expected version set and their listed status. + var expectedVersions = new Dictionary(); + foreach (var version in testCase.InitialState) + { + expectedVersions[version.ParsedVersion] = version.Data.Listed; + } + + foreach (var version in testCase.ChangesToApply) + { + if (version.IsDelete) + { + expectedVersions.Remove(version.ParsedVersion); + } + else + { + expectedVersions[version.ParsedVersion] = version.Data.Listed; + } + } + + // Verify it against the version list data. + var data = list.GetVersionListData(); + Assert.Equal( + expectedVersions.Keys.Select(x => x.ToFullString()).OrderBy(x => x).ToArray(), + data.VersionProperties.Keys.OrderBy(x => x).ToArray()); + foreach (var pair in expectedVersions) + { + Assert.True( + pair.Value == data.VersionProperties[pair.Key.ToFullString()].Listed, + $"{pair.Key.ToFullString()} should have Listed = {pair.Value} but does not."); + } + + // Verify it against the IncludePrereleaseAndSemVer2 version list, since this has all versions. + var filteredList = list._versionLists[SearchFilters.IncludePrereleaseAndSemVer2]; + Assert.Equal( + expectedVersions + .Where(x => x.Value) + .OrderBy(x => x.Key) + .Select(x => x.Key.ToFullString()) + .ToArray(), + filteredList.GetLatestVersionInfo()?.ListedFullVersions ?? new string[0]); + Assert.Equal( + expectedVersions + .Where(x => x.Value) + .Select(x => x.Key) + .OrderBy(x => x) + .LastOrDefault()? + .ToFullString(), + filteredList.GetLatestVersionInfo()?.FullVersion); + + return new TestResult(testCase, exception: null); + } + catch (Exception exception) + { + return new TestResult(testCase, exception); + } + } + + private static IEnumerable EnumerateTestCases( + IReadOnlyList fullVersions) + { + // The GetAllVersionListChanges subroutine first finds all subsets of the version set. For each subset + // of versions, enumerate all combinations of version actions for that subset. For example, consider + // the subset [ 1.0.0, 2.0.0 ] and the version actions [ Listed (L), Unlisted (U), Deleted (D) ]. + // The combinations (in no particular order) would be: + // + // [ + // [ L-1.0.0, L-2.0.0 ], [ L-1.0.0, U-2.0.0 ], [ L-1.0.0, D-2.0.0 ], + // [ U-1.0.0, L-2.0.0 ], [ U-1.0.0, U-2.0.0 ], [ U-1.0.0, D-2.0.0 ], + // [ D-1.0.0, L-2.0.0 ], [ D-1.0.0, U-2.0.0 ], [ D-1.0.0, D-2.0.0 ] + // ] + // + // The initialState sequence is this full enumeration without the Deleted version action. This is + // because a package deleted initially will simply not be present. The changesToApply sequence is this + // full enumeration with all version actions (as in the example). + // + // Finally, the cartesian produce of these two sequences is produced. Each pair is a test case. + var allActions = Enum.GetValues(typeof(VersionAction)).Cast().ToArray(); + var actionsExceptDelete = allActions.Where(x => x != VersionAction.Deleted).ToArray(); + + foreach (var initialState in GetAllVersionListChanges(fullVersions, actionsExceptDelete)) + { + foreach (var changeToApply in GetAllVersionListChanges(fullVersions, allActions)) + { + yield return new TestCase(initialState.ToArray(), changeToApply.ToList()); + } + } + } + + private static IEnumerable> GetAllVersionListChanges( + IReadOnlyCollection fullVersion, + IReadOnlyList actions) + { + foreach (var versionSubsetSequence in IterTools.SubsetsOf(fullVersion)) + { + var versionSubset = versionSubsetSequence.ToList(); + var combinations = IterTools.CombinationsOfTwo(versionSubset, actions); + foreach (var combination in combinations) + { + yield return combination.Select(x => ToVersionListChange(x.Item1, x.Item2)); + } + } + } + + private static VersionListChange ToVersionListChange(string fullVersion, VersionAction action) + { + switch (action) + { + case VersionAction.Listed: + return VersionListChange.Upsert( + fullVersion, + new VersionPropertiesData(listed: true, semVer2: false)); + case VersionAction.Unlisted: + return VersionListChange.Upsert( + fullVersion, + new VersionPropertiesData(listed: false, semVer2: false)); + case VersionAction.Deleted: + return VersionListChange.Delete(NuGetVersion.Parse(fullVersion)); + default: + throw new NotSupportedException($"The version action {action} is not supported."); + } + } + + private enum VersionAction + { + Listed, + Unlisted, + Deleted, + }; + + private class TestCase + { + public TestCase(VersionListChange[] initialState, IReadOnlyList changesToApply) + { + InitialState = initialState ?? throw new ArgumentNullException(nameof(initialState)); + ChangesToApply = changesToApply ?? throw new ArgumentNullException(nameof(changesToApply)); + } + + public VersionListChange[] InitialState { get; } + public IReadOnlyList ChangesToApply { get; } + } + + private class TestResult + { + public TestResult(TestCase testCase, Exception exception) + { + TestCase = testCase; + Exception = exception; + } + + public TestCase TestCase { get; } + public Exception Exception { get; } + } + } + + public class ApplyChanges : BaseFacts + { + [Fact] + public void ProcessesAndSolidifiesChanges() + { + var v1 = new Versions("1.0.0"); + var v2 = new Versions("2.0.0"); + var v3 = new Versions("3.0.0"); + var data = new VersionListData( + new[] { v1.Listed, v3.Listed }.ToDictionary(x => x.FullVersion, x => x.Data)); + var list = new VersionLists(data); + + var output = list.ApplyChanges(new[] { v2.Listed, v3.Unlisted }); + + Assert.Equal( + Enum.GetValues(typeof(SearchFilters)).Cast().OrderBy(x => x).ToArray(), + output.Search.Keys.OrderBy(x => x).ToArray()); + Assert.Equal(SearchIndexChangeType.UpdateLatest, output.Search[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.UpdateLatest, output.Search[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.UpdateLatest, output.Search[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.UpdateLatest, output.Search[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { v1.Parsed, v2.Parsed, v3.Parsed }, + output.Hijack.Keys.OrderBy(x => x).ToArray()); + Assert.False(output.Hijack[v1.Parsed].Delete); + Assert.False(output.Hijack[v1.Parsed].UpdateMetadata); + Assert.False(output.Hijack[v1.Parsed].LatestSemVer1); + Assert.False(output.Hijack[v2.Parsed].Delete); + Assert.True(output.Hijack[v2.Parsed].UpdateMetadata); + Assert.True(output.Hijack[v2.Parsed].LatestSemVer1); + Assert.False(output.Hijack[v3.Parsed].Delete); + Assert.True(output.Hijack[v3.Parsed].UpdateMetadata); + Assert.False(output.Hijack[v3.Parsed].LatestSemVer1); + } + } + + public class ApplyChangesInternal : BaseFacts + { + internal readonly Versions _v1; + internal readonly Versions _v2; + internal readonly Versions _v3; + internal readonly Versions _v4; + internal readonly Versions _v5; + + public ApplyChangesInternal() + { + // Use all stable, SemVer 1.0.0 packages for simplicity. Search filter predicate logic is covered by + // other tests. + _v1 = new Versions("1.0.0"); + _v2 = new Versions("2.0.0"); + _v3 = new Versions("3.0.0"); + _v4 = new Versions("4.0.0"); + _v5 = new Versions("5.0.0"); + } + + [Fact] + public void ProcessesVersionsInDescendingOrder() + { + var list = Create(_v1.Listed); + + var output = list.ApplyChangesInternal(new[] { _v2.Listed, _v3.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateLatest); + AssertHijackKeys(output, _v1, _v2, _v3); + AssertHijack(output, _v1, null, null, false); + AssertHijack(output, _v2, null, true, false); + AssertHijack(output, _v3, null, true, true); + } + + [Fact] + public void InterleavedUpsertsWithNoNewLatest() + { + var list = Create(_v1.Listed, _v3.Listed, _v5.Listed); + + var output = list.ApplyChangesInternal(new[] { _v2.Listed, _v4.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateVersionList); + AssertHijackKeys(output, _v2, _v4, _v5); + AssertHijack(output, _v2, null, true, false); + AssertHijack(output, _v4, null, true, false); + AssertHijack(output, _v5, null, null, true); + } + + [Fact] + public void InterleavedUpsertsWithNewLatest() + { + var list = Create(_v1.Listed, _v3.Listed); + + var output = list.ApplyChangesInternal(new[] { _v2.Listed, _v4.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateLatest); + AssertHijackKeys(output, _v2, _v3, _v4); + AssertHijack(output, _v2, null, true, false); + AssertHijack(output, _v3, null, null, false); + AssertHijack(output, _v4, null, true, true); + } + + [Fact] + public void InterleavedUpsertsWithNewLatestAndUnlistedHighest() + { + var list = Create(_v1.Listed, _v3.Listed, _v5.Unlisted); + + var output = list.ApplyChangesInternal(new[] { _v2.Listed, _v4.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateLatest); + AssertHijackKeys(output, _v2, _v3, _v4); + AssertHijack(output, _v2, null, true, false); + AssertHijack(output, _v3, null, null, false); + AssertHijack(output, _v4, null, true, true); + } + + [Fact] + public void InterleavedUpsertsWithRelistedLatest() + { + var list = Create(_v1.Listed, _v3.Listed, _v5.Unlisted); + + var output = list.ApplyChangesInternal(new[] { _v2.Listed, _v4.Listed, _v5.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateLatest); + AssertHijackKeys(output, _v2, _v3, _v4, _v5); + AssertHijack(output, _v2, null, true, false); + AssertHijack(output, _v3, null, null, false); + AssertHijack(output, _v4, null, true, false); + AssertHijack(output, _v5, null, true, true); + } + + [Fact] + public void RelistNewLatest() + { + var list = Create(_v1.Listed, _v2.Unlisted, _v3.Unlisted); + + var output = list.ApplyChangesInternal(new[] { _v2.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateLatest); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, null, null, false); + AssertHijack(output, _v2, null, true, true); + } + + [Fact] + public void RelistExistingLatest() + { + var list = Create(_v1.Listed); + + var output = list.ApplyChangesInternal(new[] { _v1.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateLatest); + AssertHijackKeys(output, _v1); + AssertHijack(output, _v1, null, true, true); + } + + [Fact] + public void RelistNonLatest() + { + var list = Create(_v1.Listed, _v2.Listed); + + var output = list.ApplyChangesInternal(new[] { _v1.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateVersionList); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, null, true, false); + AssertHijack(output, _v2, null, null, true); + } + + [Fact] + public void UnlistLastListed() + { + var list = Create(_v1.Unlisted, _v2.Listed, _v3.Unlisted); + + var output = list.ApplyChangesInternal(new[] { _v2.Unlisted }); + + AssertSearchFilters(output, SearchIndexChangeType.Delete); + AssertHijackKeys(output, _v2); + AssertHijack(output, _v2, null, true, false); + } + + [Fact] + public void UnlistNonLatest() + { + var list = Create(_v1.Listed, _v2.Listed, _v3.Listed); + + var output = list.ApplyChangesInternal(new[] { _v2.Unlisted, _v1.Unlisted }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateVersionList); + AssertHijackKeys(output, _v1, _v2, _v3); + AssertHijack(output, _v1, null, true, false); + AssertHijack(output, _v2, null, true, false); + AssertHijack(output, _v3, null, null, true); + } + + [Fact] + public void EmptyChangeList() + { + var list = Create(_v1.Listed, _v2.Listed, _v3.Listed); + + var output = list.ApplyChangesInternal(Enumerable.Empty()); + + Assert.Empty(output.SearchChanges); + Assert.Empty(output.HijackDocuments); + } + + [Fact] + public void DeleteNonLatestAndUpsertLatest() + { + var list = Create(_v1.Listed, _v2.Listed); + + var output = list.ApplyChangesInternal(new[] { _v1.Deleted, _v3.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateLatest); + AssertHijackKeys(output, _v1, _v2, _v3); + AssertHijack(output, _v1, true, null, null); + AssertHijack(output, _v2, null, null, false); + AssertHijack(output, _v3, null, true, true); + } + + [Fact] + public void DeleteLatestAndUpsertHigherLatest() + { + var list = Create(_v1.Listed, _v2.Listed); + + var output = list.ApplyChangesInternal(new[] { _v2.Deleted, _v3.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateLatest); + AssertHijackKeys(output, _v2, _v3); + AssertHijack(output, _v2, true, null, null); + AssertHijack(output, _v3, null, true, true); + } + + [Fact] + public void DeleteLatestAndUpsertLowerLatest() + { + var list = Create(_v1.Listed, _v3.Listed); + + var output = list.ApplyChangesInternal(new[] { _v3.Deleted, _v2.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateLatest); + AssertHijackKeys(output, _v1, _v2, _v3); + AssertHijack(output, _v1, null, null, false); + AssertHijack(output, _v2, null, true, true); + AssertHijack(output, _v3, true, null, null); + } + + [Fact] + public void DeleteLatest() + { + var list = Create(_v1.Listed, _v2.Listed); + + var output = list.ApplyChangesInternal(new[] { _v2.Deleted }); + + AssertSearchFilters(output, SearchIndexChangeType.DowngradeLatest); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, null, null, true); + AssertHijack(output, _v2, true, null, null); + } + + [Fact] + public void UnlistLatest() + { + var list = Create(_v1.Listed, _v2.Listed); + + var output = list.ApplyChangesInternal(new[] { _v2.Unlisted }); + + AssertSearchFilters(output, SearchIndexChangeType.DowngradeLatest); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, null, null, true); + AssertHijack(output, _v2, null, true, false); + } + + [Fact] + public void UnlistLatestAndRelistNewLatest() + { + var list = Create(_v1.Listed, _v2.Listed); + + var output = list.ApplyChangesInternal(new[] { _v1.Listed, _v2.Unlisted }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateLatest); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, null, true, true); + AssertHijack(output, _v2, null, true, false); + } + + [Fact] + public void DeleteLatestAndUpsertNonLatest() + { + var list = Create(_v2.Listed, _v3.Listed); + + var output = list.ApplyChangesInternal(new[] { _v3.Deleted, _v1.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.DowngradeLatest); + AssertHijackKeys(output, _v1, _v2, _v3); + AssertHijack(output, _v1, null, true, false); + AssertHijack(output, _v2, null, null, true); + AssertHijack(output, _v3, true, null, null); + } + + [Fact] + public void DeleteNonLatestAndUpsertNonLatest() + { + var list = Create(_v2.Listed, _v3.Listed); + + var output = list.ApplyChangesInternal(new[] { _v2.Deleted, _v1.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateVersionList); + AssertHijackKeys(output, _v1, _v2, _v3); + AssertHijack(output, _v1, null, true, false); + AssertHijack(output, _v2, true, null, null); + AssertHijack(output, _v3, null, null, true); + } + + [Fact] + public void DeleteLastListedWithOneRemainingUnlisted() + { + var list = Create(_v1.Unlisted, _v2.Listed); + + var output = list.ApplyChangesInternal(new[] { _v2.Deleted }); + + AssertSearchFilters(output, SearchIndexChangeType.Delete); + AssertHijackKeys(output, _v2); + AssertHijack(output, _v2, true, null, null); + } + + [Fact] + public void DeleteVeryLastWhenLastIsListed() + { + var list = Create(_v1.Listed); + + var output = list.ApplyChangesInternal(new[] { _v1.Deleted }); + + AssertSearchFilters(output, SearchIndexChangeType.Delete); + AssertHijackKeys(output, _v1); + AssertHijack(output, _v1, true, null, null); + } + + [Fact] + public void DeleteVeryLastWhenLastIsUnlisted() + { + var list = Create(_v1.Unlisted); + + var output = list.ApplyChangesInternal(new[] { _v1.Deleted }); + + AssertSearchFilters(output, SearchIndexChangeType.Delete); + AssertHijackKeys(output, _v1); + AssertHijack(output, _v1, true, null, null); + } + + [Fact] + public void AddSingleFirstWhichIsUnlisted() + { + var list = Create(); + + var output = list.ApplyChangesInternal(new[] { _v1.Unlisted }); + + AssertSearchFilters(output, SearchIndexChangeType.Delete); + AssertHijackKeys(output, _v1); + AssertHijack(output, _v1, null, true, false); + } + + [Fact] + public void AddSingleFirstWhichIsListed() + { + var list = Create(); + + var output = list.ApplyChangesInternal(new[] { _v1.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.AddFirst); + AssertHijackKeys(output, _v1); + AssertHijack(output, _v1, null, true, true); + } + + [Fact] + public void DeleteLatestAndAndNewLatestWithoutAnyOtherVersions() + { + var list = Create(_v2.Listed); + + var output = list.ApplyChangesInternal(new[] { _v1.Listed, _v2.Deleted }); + + AssertSearchFilters(output, SearchIndexChangeType.AddFirst); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, null, true, true); + AssertHijack(output, _v2, true, null, null); + } + + [Fact] + public void DeleteLatestAndAndTwoNewLatestWithoutAnyOtherVersions() + { + var list = Create(_v3.Listed); + + var output = list.ApplyChangesInternal(new[] { _v1.Listed, _v2.Listed, _v3.Deleted }); + + AssertSearchFilters(output, SearchIndexChangeType.AddFirst); + AssertHijackKeys(output, _v1, _v2, _v3); + AssertHijack(output, _v1, null, true, false); + AssertHijack(output, _v2, null, true, true); + AssertHijack(output, _v3, true, null, null); + } + + [Fact] + public void RejectsMultipleChangesForSameVersion() + { + var list = Create(); + + var ex = Assert.Throws( + () => list.ApplyChangesInternal(new[] + { + _v1.Listed, + _v1.Unlisted, + _v2.Listed, + _v2.Listed, + _v2.Listed, + })); + Assert.Contains( + "There are multiple changes for the following version(s): 1.0.0 (2 changes), 2.0.0 (3 changes)", + ex.Message); + } + + [Fact] + public void AddMultipleFirstWhenAllAreListed() + { + var list = Create(); + + var output = list.ApplyChangesInternal(new[] { _v1.Listed, _v2.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.AddFirst); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, null, true, false); + AssertHijack(output, _v2, null, true, true); + } + + [Fact] + public void AddMultipleFirstWhenAllAreUnlisted() + { + var list = Create(); + + var output = list.ApplyChangesInternal(new[] { _v1.Unlisted, _v2.Unlisted }); + + AssertSearchFilters(output, SearchIndexChangeType.Delete); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, null, true, false); + AssertHijack(output, _v2, null, true, false); + } + + [Fact] + public void AddMultipleFirstWhenWithListedLessThanUnlisted() + { + var list = Create(); + + var output = list.ApplyChangesInternal(new[] { _v1.Listed, _v2.Unlisted }); + + AssertSearchFilters(output, SearchIndexChangeType.AddFirst); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, null, true, true); + AssertHijack(output, _v2, null, true, false); + } + + [Fact] + public void AddMultipleFirstWhenWithListedGreaterThanUnlisted() + { + var list = Create(); + + var output = list.ApplyChangesInternal(new[] { _v1.Unlisted, _v2.Listed }); + + AssertSearchFilters(output, SearchIndexChangeType.AddFirst); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, null, true, false); + AssertHijack(output, _v2, null, true, true); + } + + [Fact] + public void DeleteNonExistingVersionFromEmptyList() + { + var list = Create(); + + var output = list.ApplyChangesInternal(new[] { _v1.Deleted }); + + AssertSearchFilters(output, SearchIndexChangeType.Delete); + AssertHijackKeys(output, _v1); + AssertHijack(output, _v1, true, null, null); + } + + [Fact] + public void DeleteNonExistingVersionFromListWithLatest() + { + var list = Create(_v1.Listed); + + var output = list.ApplyChangesInternal(new[] { _v2.Deleted }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateVersionList); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, null, null, true); + AssertHijack(output, _v2, true, null, null); + } + + [Fact] + public void DeleteNonExistingVersionFromListWithOnlyUnlisted() + { + var list = Create(_v1.Unlisted); + + var output = list.ApplyChangesInternal(new[] { _v2.Deleted }); + + AssertSearchFilters(output, SearchIndexChangeType.Delete); + AssertHijackKeys(output, _v2); + AssertHijack(output, _v2, true, null, null); + } + + [Fact] + public void DeleteNonExistingVersionAndAddNewVersion() + { + var list = Create(_v1.Listed); + + var output = list.ApplyChangesInternal(new[] { _v2.Listed, _v3.Deleted }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateLatest); + AssertHijackKeys(output, _v1, _v2, _v3); + AssertHijack(output, _v1, null, null, false); + AssertHijack(output, _v2, null, true, true); + AssertHijack(output, _v3, true, null, null); + } + + [Fact] + public void UnlistLatestAndDeleteNextLatest() + { + var list = Create(_v1.Listed, _v2.Listed); + + var output = list.ApplyChangesInternal(new[] { _v1.Deleted, _v2.Unlisted }); + + AssertSearchFilters(output, SearchIndexChangeType.Delete); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, true, null, null); + AssertHijack(output, _v2, null, true, false); + } + + [Fact] + public void DeleteNonExistentAndUnlistLatest() + { + var list = Create(_v1.Listed); + + var output = list.ApplyChangesInternal(new[] { _v1.Unlisted, _v2.Deleted }); + + AssertSearchFilters(output, SearchIndexChangeType.Delete); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, null, true, false); + AssertHijack(output, _v2, true, null, null); + } + + [Fact] + public void UnlistNewHighestVersionAndDeleteLatest() + { + var list = Create(_v1.Listed); + + var output = list.ApplyChangesInternal(new[] { _v1.Deleted, _v2.Unlisted }); + + AssertSearchFilters(output, SearchIndexChangeType.Delete); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, true, null, null); + AssertHijack(output, _v2, null, true, false); + } + + [Fact] + public void UnlistNewHighestVersionAndUnlistLatest() + { + var list = Create(_v1.Listed, _v2.Listed); + + var output = list.ApplyChangesInternal(new[] { _v2.Unlisted, _v3.Unlisted }); + + AssertSearchFilters(output, SearchIndexChangeType.DowngradeLatest); + AssertHijackKeys(output, _v1, _v2, _v3); + AssertHijack(output, _v1, null, null, true); + AssertHijack(output, _v2, null, true, false); + AssertHijack(output, _v2, null, true, false); + } + + [Fact] + public void AddUnlistedHighestThenNewLatest() + { + var list = Create(_v1.Listed); + + var output = list.ApplyChangesInternal(new[] { _v2.Listed, _v3.Unlisted }); + + AssertSearchFilters(output, SearchIndexChangeType.UpdateLatest); + AssertHijackKeys(output, _v1, _v2, _v3); + AssertHijack(output, _v1, null, null, false); + AssertHijack(output, _v2, null, true, true); + AssertHijack(output, _v3, null, true, false); + } + + [Fact] + public void AddUnlistedHighestThenFirstLatest() + { + var list = Create(); + + var output = list.ApplyChangesInternal(new[] { _v1.Listed, _v2.Unlisted }); + + AssertSearchFilters(output, SearchIndexChangeType.AddFirst); + AssertHijackKeys(output, _v1, _v2); + AssertHijack(output, _v1, null, true, true); + AssertHijack(output, _v2, null, true, false); + } + + private void AssertSearchFilters(MutableIndexChanges output, SearchIndexChangeType type) + { + Assert.Equal(type, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(type, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(type, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(type, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + } + + private void AssertHijackKeys(MutableIndexChanges output, params Versions[] versions) + { + Assert.Equal( + versions.Select(x => x.Parsed).OrderBy(x => x).ToArray(), + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + } + + private void AssertHijack(MutableIndexChanges output, Versions versions, bool? delete, bool? updateMetadata, bool? latest) + { + Assert.Equal(delete, output.HijackDocuments[versions.Parsed].Delete); + Assert.Equal(updateMetadata, output.HijackDocuments[versions.Parsed].UpdateMetadata); + Assert.Equal(latest, output.HijackDocuments[versions.Parsed].LatestStableSemVer1); + Assert.Equal(latest, output.HijackDocuments[versions.Parsed].LatestSemVer1); + Assert.Equal(latest, output.HijackDocuments[versions.Parsed].LatestStableSemVer2); + Assert.Equal(latest, output.HijackDocuments[versions.Parsed].LatestSemVer2); + } + + internal static VersionLists Create(params VersionListChange[] versions) + { + if (versions.Any(x => x.IsDelete)) + { + throw new ArgumentException(nameof(versions)); + } + + var data = new VersionListData(versions.ToDictionary(x => x.FullVersion, x => x.Data)); + return new VersionLists(data); + } + } + + public class Upsert : BaseFacts + { + [Fact] + public void ReplacesLatestFullVersionByNormalizedVersion() + { + var list = Create( + new VersionProperties("1.02.0-Alpha.1+git", new VersionPropertiesData(listed: true, semVer2: true))); + + list.Upsert("1.2.0.0-ALPHA.1+somethingelse", new VersionPropertiesData(listed: true, semVer2: true)); + + Assert.Equal( + new[] { "1.2.0-ALPHA.1+somethingelse" }, + list.GetVersionListData().VersionProperties.Keys.ToArray()); + } + + [Fact] + public void ReplacesNonLatestFullVersionByNormalizedVersion() + { + var list = Create( + new VersionProperties("2.0.0", new VersionPropertiesData(listed: true, semVer2: true)), + new VersionProperties("1.02.0-Alpha.1+git", new VersionPropertiesData(listed: true, semVer2: true))); + + list.Upsert("1.2.0.0-ALPHA.1+somethingelse", new VersionPropertiesData(listed: true, semVer2: true)); + + Assert.Equal( + new[] { "1.2.0-ALPHA.1+somethingelse", "2.0.0" }, + list.GetVersionListData().VersionProperties.Keys.ToArray()); + } + + [Fact] + public void DifferentUpdateLatestForAllFilters() + { + var list = Create(_stableSemVer1Listed, _prereleaseSemVer1Listed, _stableSemVer2Listed, _prereleaseSemVer2Listed); + var latest = new VersionProperties("5.0.0", new VersionPropertiesData(listed: true, semVer2: false)); + + var output = list.Upsert(latest); + + Assert.Equal(SearchIndexChangeType.UpdateLatest, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.UpdateLatest, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.UpdateLatest, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.UpdateLatest, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] + { + _stableSemVer1Listed.ParsedVersion, + _prereleaseSemVer1Listed.ParsedVersion, + _stableSemVer2Listed.ParsedVersion, + _prereleaseSemVer2Listed.ParsedVersion, + latest.ParsedVersion, + }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: null, + latestStableSemVer1: false, + latestSemVer1: null, + latestStableSemVer2: null, + latestSemVer2: null), + output.HijackDocuments[_stableSemVer1Listed.ParsedVersion]); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: null, + latestStableSemVer1: null, + latestSemVer1: false, + latestStableSemVer2: null, + latestSemVer2: null), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: null, + latestStableSemVer1: null, + latestSemVer1: null, + latestStableSemVer2: false, + latestSemVer2: null), + output.HijackDocuments[_stableSemVer2Listed.ParsedVersion]); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: null, + latestStableSemVer1: null, + latestSemVer1: null, + latestStableSemVer2: null, + latestSemVer2: false), + output.HijackDocuments[_prereleaseSemVer2Listed.ParsedVersion]); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: true, + latestStableSemVer1: true, + latestSemVer1: true, + latestStableSemVer2: true, + latestSemVer2: true), + output.HijackDocuments[latest.ParsedVersion]); + } + + [Fact] + public void AddPartiallyApplicableLatestVersion() + { + var list = Create(_stableSemVer1Listed); + + var output = list.Upsert(_prereleaseSemVer1Listed); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.UpdateLatest, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.UpdateLatest, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _stableSemVer1Listed.ParsedVersion, _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: null, + latestStableSemVer1: true, + latestSemVer1: false, + latestStableSemVer2: true, + latestSemVer2: false), + output.HijackDocuments[_stableSemVer1Listed.ParsedVersion]); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: true, + latestStableSemVer1: false, + latestSemVer1: true, + latestStableSemVer2: false, + latestSemVer2: true), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void AddPartiallyApplicableLatestUnlistedVersion() + { + var list = Create(_stableSemVer1Listed); + + var output = list.Upsert(_prereleaseSemVer1Unlisted); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _stableSemVer1Listed.ParsedVersion, _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: null, + latestStableSemVer1: true, + latestSemVer1: true, + latestStableSemVer2: true, + latestSemVer2: true), + output.HijackDocuments[_stableSemVer1Listed.ParsedVersion]); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: true, + latestStableSemVer1: false, + latestSemVer1: false, + latestStableSemVer2: false, + latestSemVer2: false), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void AddPartiallyApplicableNonLatestVersion() + { + var list = Create(_prereleaseSemVer1Listed); + + var output = list.Upsert(_stableSemVer1Listed); + + Assert.Equal(SearchIndexChangeType.AddFirst, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.AddFirst, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _stableSemVer1Listed.ParsedVersion, _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: true, + latestStableSemVer1: true, + latestSemVer1: false, + latestStableSemVer2: true, + latestSemVer2: false), + output.HijackDocuments[_stableSemVer1Listed.ParsedVersion]); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: null, + latestStableSemVer1: null, + latestSemVer1: true, + latestStableSemVer2: null, + latestSemVer2: true), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void AddPartiallyApplicableNonLatestUnlistedVersion() + { + var list = Create(_prereleaseSemVer1Listed); + + var output = list.Upsert(_stableSemVer1Unlisted); + + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _stableSemVer1Listed.ParsedVersion, _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: true, + latestStableSemVer1: false, + latestSemVer1: false, + latestStableSemVer2: false, + latestSemVer2: false), + output.HijackDocuments[_stableSemVer1Listed.ParsedVersion]); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: null, + latestStableSemVer1: null, + latestSemVer1: true, + latestStableSemVer2: null, + latestSemVer2: true), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void AddPartiallyApplicableLatestVersionWhenOnlyUnlistedExists() + { + var list = Create(_stableSemVer1Unlisted); + + var output = list.Upsert(_prereleaseSemVer1Listed); + + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.AddFirst, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.AddFirst, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: true, + latestStableSemVer1: false, + latestSemVer1: true, + latestStableSemVer2: false, + latestSemVer2: true), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void AddPartiallyApplicableLatestUnlistedVersionWhenOnlyUnlistedExists() + { + var list = Create(_stableSemVer1Unlisted); + + var output = list.Upsert(_prereleaseSemVer1Unlisted); + + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: true, + latestStableSemVer1: false, + latestSemVer1: false, + latestStableSemVer2: false, + latestSemVer2: false), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void AddPartiallyApplicableNonLatestVersionWhenOnlyUnlistedExists() + { + var list = Create(_prereleaseSemVer1Unlisted); + + var output = list.Upsert(_stableSemVer1Listed); + + Assert.Equal(SearchIndexChangeType.AddFirst, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.AddFirst, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.AddFirst, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.AddFirst, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _stableSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: true, + latestStableSemVer1: true, + latestSemVer1: true, + latestStableSemVer2: true, + latestSemVer2: true), + output.HijackDocuments[_stableSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void AddPartiallyApplicableNonLatestUnlistedVersionWhenOnlyUnlistedExists() + { + var list = Create(_prereleaseSemVer1Unlisted); + + var output = list.Upsert(_stableSemVer1Unlisted); + + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _stableSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: true, + latestStableSemVer1: false, + latestSemVer1: false, + latestStableSemVer2: false, + latestSemVer2: false), + output.HijackDocuments[_stableSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void UnlistLatestWhenOtherVersionExists() + { + var list = Create(_stableSemVer1Listed, _prereleaseSemVer1Listed); + + var output = list.Upsert(_prereleaseSemVer1Unlisted); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.DowngradeLatest, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.DowngradeLatest, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _stableSemVer1Listed.ParsedVersion, _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: null, + latestStableSemVer1: true, + latestSemVer1: true, + latestStableSemVer2: true, + latestSemVer2: true), + output.HijackDocuments[_stableSemVer1Listed.ParsedVersion]); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: true, + latestStableSemVer1: false, + latestSemVer1: false, + latestStableSemVer2: false, + latestSemVer2: false), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void UnlistLatestWhenNoOtherVersionExists() + { + var list = Create(_prereleaseSemVer1Listed); + + var output = list.Upsert(_prereleaseSemVer1Unlisted); + + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: true, + latestStableSemVer1: false, + latestSemVer1: false, + latestStableSemVer2: false, + latestSemVer2: false), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void UnlistLatestWhenOnlyUnlistOtherVersionExists() + { + var list = Create(_stableSemVer1Unlisted, _prereleaseSemVer1Listed); + + var output = list.Upsert(_prereleaseSemVer1Unlisted); + + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: true, + latestStableSemVer1: false, + latestSemVer1: false, + latestStableSemVer2: false, + latestSemVer2: false), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void UnlistNonLatestWhenLatestExists() + { + var list = Create(_stableSemVer1Listed, _prereleaseSemVer1Listed); + + var output = list.Upsert(_stableSemVer1Unlisted); + + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _stableSemVer1Listed.ParsedVersion, _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: true, + latestStableSemVer1: false, + latestSemVer1: false, + latestStableSemVer2: false, + latestSemVer2: false), + output.HijackDocuments[_stableSemVer1Listed.ParsedVersion]); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: null, + latestStableSemVer1: null, + latestSemVer1: true, + latestStableSemVer2: null, + latestSemVer2: true), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + } + + public class Delete : BaseFacts + { + [Fact] + public void DeletesByNormalizedVersion() + { + var list = Create( + new VersionProperties("1.02.0-Alpha.1+git", new VersionPropertiesData(listed: true, semVer2: true))); + + list.Delete("1.2.0.0-ALPHA.1+somethingelse"); + + Assert.Empty(list.GetVersionListData().VersionProperties); + } + + [Fact] + public void DeleteUnknownVersionWhenSingleListedVersionExists() + { + var list = Create(_prereleaseSemVer1Listed); + + var output = list.Delete(StableSemVer1); + + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _stableSemVer1Listed.ParsedVersion, _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: true, + updateMetadata: null, + latestStableSemVer1: null, + latestSemVer1: null, + latestStableSemVer2: null, + latestSemVer2: null), + output.HijackDocuments[_stableSemVer1Listed.ParsedVersion]); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: null, + latestStableSemVer1: null, + latestSemVer1: true, + latestStableSemVer2: null, + latestSemVer2: true), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void DeleteUnknownVersionWhenSingleUnlistedVersionExists() + { + var list = Create(_stableSemVer1Unlisted); + + var output = list.Delete(PrereleaseSemVer1); + + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: true, + updateMetadata: null, + latestStableSemVer1: null, + latestSemVer1: null, + latestStableSemVer2: null, + latestSemVer2: null), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void DeleteLatestVersionWhenSingleListedVersionExists() + { + var list = Create(_prereleaseSemVer1Listed); + + var output = list.Delete(PrereleaseSemVer1); + + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: true, + updateMetadata: null, + latestStableSemVer1: null, + latestSemVer1: null, + latestStableSemVer2: null, + latestSemVer2: null), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void DeleteLatestVersionWhenTwoListedVersionsExists() + { + var list = Create(_stableSemVer1Listed, _prereleaseSemVer1Listed); + + var output = list.Delete(PrereleaseSemVer1); + + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.DowngradeLatest, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.DowngradeLatest, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _stableSemVer1Listed.ParsedVersion, _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: null, + latestStableSemVer1: true, + latestSemVer1: true, + latestStableSemVer2: true, + latestSemVer2: true), + output.HijackDocuments[_stableSemVer1Listed.ParsedVersion]); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: true, + updateMetadata: null, + latestStableSemVer1: null, + latestSemVer1: null, + latestStableSemVer2: null, + latestSemVer2: null), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + + [Fact] + public void DeleteNonLatestVersionWhenTwoListedVersionsExists() + { + var list = Create(_stableSemVer1Listed, _prereleaseSemVer1Listed); + + var output = list.Delete(StableSemVer1); + + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.Default]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludePrerelease]); + Assert.Equal(SearchIndexChangeType.Delete, output.SearchChanges[SearchFilters.IncludeSemVer2]); + Assert.Equal(SearchIndexChangeType.UpdateVersionList, output.SearchChanges[SearchFilters.IncludePrereleaseAndSemVer2]); + Assert.Equal( + new[] { _stableSemVer1Listed.ParsedVersion, _prereleaseSemVer1Listed.ParsedVersion }, + output.HijackDocuments.Keys.OrderBy(x => x).ToArray()); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: true, + updateMetadata: null, + latestStableSemVer1: null, + latestSemVer1: null, + latestStableSemVer2: null, + latestSemVer2: null), + output.HijackDocuments[_stableSemVer1Listed.ParsedVersion]); + Assert.Equal( + new MutableHijackDocumentChanges( + delete: null, + updateMetadata: null, + latestStableSemVer1: null, + latestSemVer1: true, + latestStableSemVer2: null, + latestSemVer2: true), + output.HijackDocuments[_prereleaseSemVer1Listed.ParsedVersion]); + } + } + + public abstract class BaseFacts + { + internal const string StableSemVer1 = "1.0.0"; + internal const string PrereleaseSemVer1 = "2.0.0-alpha"; + internal const string StableSemVer2 = "3.0.0"; + internal const string PrereleaseSemVer2 = "4.0.0-alpha"; + + internal readonly VersionProperties _stableSemVer1Listed; + internal readonly VersionProperties _stableSemVer1Unlisted; + internal readonly VersionProperties _prereleaseSemVer1Listed; + internal readonly VersionProperties _prereleaseSemVer1Unlisted; + internal readonly VersionProperties _stableSemVer2Listed; + internal readonly VersionProperties _stableSemVer2Unlisted; + internal readonly VersionProperties _prereleaseSemVer2Listed; + internal readonly VersionProperties _prereleaseSemVer2Unlisted; + + protected BaseFacts() + { + _stableSemVer1Listed = Create(StableSemVer1, true, false); + _stableSemVer1Unlisted = Create(StableSemVer1, false, false); + _prereleaseSemVer1Listed = Create(PrereleaseSemVer1, true, false); + _prereleaseSemVer1Unlisted = Create(PrereleaseSemVer1, false, false); + _stableSemVer2Listed = Create(StableSemVer2, true, true); + _stableSemVer2Unlisted = Create(StableSemVer2, false, true); + _prereleaseSemVer2Listed = Create(PrereleaseSemVer2, true, true); + _prereleaseSemVer2Unlisted = Create(PrereleaseSemVer2, false, true); + } + + private VersionProperties Create(string version, bool listed, bool semVer2) + { + return new VersionProperties(version, new VersionPropertiesData(listed, semVer2)); + } + + internal VersionLists Create(params VersionProperties[] versions) + { + var data = new VersionListData(versions.ToDictionary(x => x.FullVersion, x => x.Data)); + return new VersionLists(data); + } + + internal class Versions + { + public Versions(string fullOrOriginalVersion) + { + Listed = VersionListChange.Upsert(fullOrOriginalVersion, new VersionPropertiesData(listed: true, semVer2: false)); + Full = Listed.FullVersion; + Parsed = Listed.ParsedVersion; + Unlisted = VersionListChange.Upsert(fullOrOriginalVersion, new VersionPropertiesData(listed: false, semVer2: false)); + Deleted = VersionListChange.Delete(Listed.ParsedVersion); + Deleted = VersionListChange.Delete(Listed.ParsedVersion); + } + + public string Full { get; } + public NuGetVersion Parsed { get; } + public VersionListChange Listed { get; } + public VersionListChange Unlisted { get; } + public VersionListChange Deleted { get; } + } + } + } +} diff --git a/tests/NuGet.Services.AzureSearch.Tests/Wrappers/DocumentOperationsWrapperFacts.cs b/tests/NuGet.Services.AzureSearch.Tests/Wrappers/DocumentOperationsWrapperFacts.cs new file mode 100644 index 000000000..754439773 --- /dev/null +++ b/tests/NuGet.Services.AzureSearch.Tests/Wrappers/DocumentOperationsWrapperFacts.cs @@ -0,0 +1,255 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Search; +using Microsoft.Azure.Search.Models; +using Microsoft.Rest; +using Microsoft.Rest.Azure; +using Moq; +using Moq.Language; +using NuGet.Services.AzureSearch.SearchService; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Services.AzureSearch.Wrappers +{ + public class DocumentOperationsWrapperFacts + { + public class IndexAsync : Facts + { + public IndexAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task DoesNotHandleNotFoundException() + { + DocumentOperations + .Setup(x => x.IndexWithHttpMessagesAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny>>(), + It.IsAny())) + .ThrowsAsync(new CloudException + { + Response = new HttpResponseMessageWrapper(new HttpResponseMessage(HttpStatusCode.NotFound), string.Empty), + }); + + await Assert.ThrowsAsync( + () => Target.IndexAsync(new IndexBatch(Enumerable.Empty>()))); + } + + [Fact] + public async Task DoesNotWrapIndexBatchException() + { + DocumentOperations + .Setup(x => x.IndexWithHttpMessagesAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny>>(), + It.IsAny())) + .ThrowsAsync(new IndexBatchException(new DocumentIndexResult(new List()))); + + await Assert.ThrowsAsync( + () => Target.IndexAsync(new IndexBatch(Enumerable.Empty>()))); + } + + [Fact] + public async Task DoesNotRetryOnNullReferenceException() + { + DocumentOperations + .Setup(x => x.IndexWithHttpMessagesAsync( + It.IsAny>(), + It.IsAny(), + It.IsAny>>(), + It.IsAny())) + .ThrowsAsync(new NullReferenceException()); + + var ex = await Assert.ThrowsAsync( + () => Target.IndexAsync(new IndexBatch(Enumerable.Empty>()))); + + Assert.Equal(1, DocumentOperations.Invocations.Count); + } + } + + public class GetOrNullAsyncOfT : RetryFacts + { + public GetOrNullAsyncOfT(ITestOutputHelper output) : base(output) + { + } + + public override bool TreatsNotFoundAsDefault => true; + + public override async Task ExecuteAsync() + { + return await Target.GetOrNullAsync(string.Empty); + } + + public override IReturns>> Setup() + { + return DocumentOperations + .Setup(x => x.GetWithHttpMessagesAsync( + It.IsAny(), + It.IsAny>(), + It.IsAny(), + It.IsAny>>(), + It.IsAny())); + } + } + + public class SearchAsync : RetryFacts + { + public SearchAsync(ITestOutputHelper output) : base(output) + { + } + + public override bool TreatsNotFoundAsDefault => false; + + public override async Task ExecuteAsync() + { + return await Target.SearchAsync(string.Empty, new SearchParameters()); + } + + public override IReturns>> Setup() + { + return DocumentOperations + .Setup(x => x.SearchWithHttpMessagesAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>>(), + It.IsAny())); + } + } + + public class SearchAsyncOfT : RetryFacts> + { + public SearchAsyncOfT(ITestOutputHelper output) : base(output) + { + } + + public override bool TreatsNotFoundAsDefault => false; + + public override async Task> ExecuteAsync() + { + return await Target.SearchAsync(string.Empty, new SearchParameters()); + } + + public override IReturns>>> Setup() + { + return DocumentOperations + .Setup(x => x.SearchWithHttpMessagesAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>>(), + It.IsAny())); + } + } + + public class CountAsync : RetryFacts + { + public CountAsync(ITestOutputHelper output) : base(output) + { + } + + public override bool TreatsNotFoundAsDefault => false; + + public override async Task ExecuteAsync() + { + return await Target.CountAsync(); + } + + public override IReturns>> Setup() + { + return DocumentOperations + .Setup(x => x.CountWithHttpMessagesAsync( + It.IsAny(), + It.IsAny>>(), + It.IsAny())); + } + } + + public abstract class RetryFacts : Facts + { + public RetryFacts(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task HandlesNotFoundException() + { + Setup() + .ThrowsAsync(new CloudException + { + Response = new HttpResponseMessageWrapper(new HttpResponseMessage(HttpStatusCode.NotFound), string.Empty), + }); + + if (TreatsNotFoundAsDefault) + { + var result = await ExecuteAsync(); + Assert.Equal(default(T), result); + } + else + { + await Assert.ThrowsAsync(() => ExecuteAsync()); + } + } + + [Fact] + public async Task RethrowsBadRequest() + { + Setup() + .ThrowsAsync(new CloudException + { + Response = new HttpResponseMessageWrapper(new HttpResponseMessage(HttpStatusCode.BadRequest), string.Empty), + }); + + var ex = await Assert.ThrowsAsync(() => ExecuteAsync()); + Assert.Equal("The provided query is invalid.", ex.Message); + Assert.IsType(ex.InnerException); + } + + [Fact] + public async Task RetriesOnNullReferenceException() + { + Setup() + .ThrowsAsync(new NullReferenceException()); + + var ex = await Assert.ThrowsAsync(() => ExecuteAsync()); + + Assert.Equal(3, DocumentOperations.Invocations.Count); + Assert.Equal(2, Logger.Messages.Count); + Assert.Equal("The search query failed due to Azure/azure-sdk-for-net#3224.", ex.Message); + } + + public abstract bool TreatsNotFoundAsDefault { get; } + public abstract IReturns>> Setup(); + public abstract Task ExecuteAsync(); + } + + public abstract class Facts + { + public Facts(ITestOutputHelper output) + { + DocumentOperations = new Mock(); + Logger = output.GetLogger(); + + Target = new DocumentsOperationsWrapper( + DocumentOperations.Object, + Logger); + } + + public Mock DocumentOperations { get; } + public RecordingLogger Logger { get; } + public DocumentsOperationsWrapper Target { get; } + } + } +} diff --git a/tests/NuGet.Services.SearchService.Tests/App_Start/ApiExceptionFilterAttributeFacts.cs b/tests/NuGet.Services.SearchService.Tests/App_Start/ApiExceptionFilterAttributeFacts.cs new file mode 100644 index 000000000..32bf3bd68 --- /dev/null +++ b/tests/NuGet.Services.SearchService.Tests/App_Start/ApiExceptionFilterAttributeFacts.cs @@ -0,0 +1,86 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using System.Web.Http; +using System.Web.Http.Controllers; +using System.Web.Http.Filters; +using System.Web.Http.Hosting; +using Moq; +using Newtonsoft.Json.Linq; +using NuGet.Services.AzureSearch; +using NuGet.Services.AzureSearch.SearchService; +using Xunit; + +namespace NuGet.Services.SearchService +{ + public class ApiExceptionFilterAttributeFacts + { + [Fact] + public void DoesNothingForUnknownException() + { + var target = new ApiExceptionFilterAttribute(); + var context = GetContext(new InvalidOperationException("Something weird.")); + + target.OnException(context); + + Assert.Null(context.Response); + } + + [Theory] + [MemberData(nameof(KnownExceptions))] + public async Task ReturnsExpectedStatusCodeForKnowExceptions(Exception ex, HttpStatusCode statusCode, string message) + { + var target = new ApiExceptionFilterAttribute(); + var context = GetContext(ex); + + target.OnException(context); + + Assert.NotNull(context.Response); + Assert.Equal(statusCode, context.Response.StatusCode); + var content = await context.Response.Content.ReadAsStringAsync(); + var json = JObject.Parse(content); + Assert.Equal(new[] { "Success", "Message" }, json.Properties().Select(x => x.Name).ToArray()); + Assert.Equal(false, json["Success"]); + Assert.Equal(message, json["Message"]); + } + + private static HttpActionExecutedContext GetContext(Exception ex) + { + var httpControllerContext = new HttpControllerContext + { + Request = new HttpRequestMessage(HttpMethod.Get, "https://example/query") + { + Properties = + { + { HttpPropertyKeys.HttpConfigurationKey, new HttpConfiguration() }, + }, + }, + }; + var httpActionContext = new HttpActionContext(httpControllerContext, actionDescriptor: Mock.Of()); + var context = new HttpActionExecutedContext(httpActionContext, ex); + return context; + } + + public static IEnumerable KnownExceptions => new[] + { + new object[] + { + new AzureSearchException("Azure Search died!", null), + HttpStatusCode.ServiceUnavailable, + "The service is unavailable.", + }, + new object[] + { + new InvalidSearchRequestException("Bad!"), + HttpStatusCode.BadRequest, + "Bad!", + }, + }; + } +} diff --git a/tests/NuGet.Services.SearchService.Tests/Controllers/SearchControllerFacts.cs b/tests/NuGet.Services.SearchService.Tests/Controllers/SearchControllerFacts.cs new file mode 100644 index 000000000..8126b7a33 --- /dev/null +++ b/tests/NuGet.Services.SearchService.Tests/Controllers/SearchControllerFacts.cs @@ -0,0 +1,597 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Reflection; +using System.Threading.Tasks; +using System.Web.Http; +using Moq; +using NuGet.Services.AzureSearch.SearchService; +using Xunit; + +namespace NuGet.Services.SearchService.Controllers +{ + public class SearchControllerFacts + { + public class IndexAsync : BaseFacts + { + private readonly HttpRequestMessage _request; + private readonly SearchStatusResponse _status; + + public IndexAsync() + { + _request = new HttpRequestMessage(); + _request.SetConfiguration(new HttpConfiguration()); + + _status = new SearchStatusResponse + { + Success = true, + Duration = TimeSpan.FromTicks(123), + }; + + _statusService + .Setup(x => x.GetStatusAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => _status); + } + + [Fact] + public async Task DoesNotInitializeAuxiliaryDataCache() + { + await _target.IndexAsync(_request); + + _auxiliaryDataCache.Verify(x => x.EnsureInitializedAsync(), Times.Never); + } + + [Fact] + public async Task PassesAllOptionsAndControllersAssembly() + { + await _target.IndexAsync(_request); + + _statusService.Verify(x => x.GetStatusAsync(SearchStatusOptions.All, _target.GetType().Assembly), Times.Once); + _statusService.Verify(x => x.GetStatusAsync(It.IsAny(), It.IsAny()), Times.Once); + } + + [Theory] + [InlineData(false, HttpStatusCode.InternalServerError)] + [InlineData(true, HttpStatusCode.OK)] + public async Task ReturnsProperStatusCode(bool success, HttpStatusCode expected) + { + _status.Success = success; + + var response = await _target.IndexAsync(_request); + + Assert.Equal(expected, response.StatusCode); + var content = Assert.IsType>(response.Content); + Assert.NotSame(_status, content.Value); + var status = Assert.IsType(content.Value); + Assert.Null(status.Duration); + } + } + + public class GetStatusAsync : BaseFacts + { + private readonly HttpRequestMessage _request; + private readonly SearchStatusResponse _status; + + public GetStatusAsync() + { + _request = new HttpRequestMessage(); + _request.SetConfiguration(new HttpConfiguration()); + + _status = new SearchStatusResponse { Success = true }; + + _statusService + .Setup(x => x.GetStatusAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(() => _status); + } + + [Fact] + public async Task DoesNotInitializeAuxiliaryDataCache() + { + await _target.GetStatusAsync(_request); + + _auxiliaryDataCache.Verify(x => x.EnsureInitializedAsync(), Times.Never); + } + + [Fact] + public async Task PassesAllOptionsAndControllersAssembly() + { + await _target.GetStatusAsync(_request); + + _statusService.Verify(x => x.GetStatusAsync(SearchStatusOptions.All, _target.GetType().Assembly), Times.Once); + _statusService.Verify(x => x.GetStatusAsync(It.IsAny(), It.IsAny()), Times.Once); + } + + [Theory] + [InlineData(false, HttpStatusCode.InternalServerError)] + [InlineData(true, HttpStatusCode.OK)] + public async Task ReturnsProperStatusCode(bool success, HttpStatusCode expected) + { + _status.Success = success; + + var response = await _target.GetStatusAsync(_request); + + Assert.Equal(expected, response.StatusCode); + var content = Assert.IsType>(response.Content); + Assert.Same(_status, content.Value); + } + } + + public class V2SearchAsync : BaseFacts + { + [Fact] + public async Task InitializesAuxiliaryDataCache() + { + await _target.V2SearchAsync(); + + _auxiliaryDataCache.Verify(x => x.EnsureInitializedAsync(), Times.Once); + } + + [Fact] + public async Task HasDefaultParameters() + { + V2SearchRequest lastRequest = null; + _searchService + .Setup(x => x.V2SearchAsync(It.IsAny())) + .ReturnsAsync(() => _v2SearchResponse) + .Callback(x => lastRequest = x); + + await _target.V2SearchAsync(); + + _searchService.Verify(x => x.V2SearchAsync(It.IsAny()), Times.Once); + Assert.NotNull(lastRequest); + Assert.Equal(0, lastRequest.Skip); + Assert.Equal(20, lastRequest.Take); + Assert.False(lastRequest.IgnoreFilter); + Assert.False(lastRequest.CountOnly); + Assert.False(lastRequest.IncludePrerelease); + Assert.False(lastRequest.IncludeSemVer2); + Assert.Null(lastRequest.Query); + Assert.True(lastRequest.LuceneQuery); + Assert.False(lastRequest.ShowDebug); + Assert.Null(lastRequest.PackageType); + } + + [Fact] + public async Task SupportsNullParameters() + { + V2SearchRequest lastRequest = null; + _searchService + .Setup(x => x.V2SearchAsync(It.IsAny())) + .ReturnsAsync(() => _v2SearchResponse) + .Callback(x => lastRequest = x); + + await _target.V2SearchAsync( + skip: null, + take: null, + ignoreFilter: null, + countOnly: null, + prerelease: null, + semVerLevel: null, + q: null, + sortBy: null, + luceneQuery: null, + debug: null, + packageType: null); + + _searchService.Verify(x => x.V2SearchAsync(It.IsAny()), Times.Once); + Assert.NotNull(lastRequest); + Assert.Equal(0, lastRequest.Skip); + Assert.Equal(20, lastRequest.Take); + Assert.False(lastRequest.IgnoreFilter); + Assert.False(lastRequest.CountOnly); + Assert.False(lastRequest.IncludePrerelease); + Assert.False(lastRequest.IncludeSemVer2); + Assert.Null(lastRequest.Query); + Assert.True(lastRequest.LuceneQuery); + Assert.False(lastRequest.ShowDebug); + Assert.Null(lastRequest.PackageType); + } + + [Fact] + public async Task UsesProvidedParameters() + { + V2SearchRequest lastRequest = null; + _searchService + .Setup(x => x.V2SearchAsync(It.IsAny())) + .ReturnsAsync(() => _v2SearchResponse) + .Callback(x => lastRequest = x); + + await _target.V2SearchAsync( + skip: -20, + take: 30000, + ignoreFilter: true, + countOnly: true, + prerelease: true, + semVerLevel: "2.0.0", + q: "windows azure storage", + sortBy: "lastEdited", + luceneQuery: true, + debug: true, + packageType: "dotnettool"); + + _searchService.Verify(x => x.V2SearchAsync(It.IsAny()), Times.Once); + Assert.NotNull(lastRequest); + Assert.Equal(-20, lastRequest.Skip); + Assert.Equal(30000, lastRequest.Take); + Assert.True(lastRequest.IgnoreFilter); + Assert.True(lastRequest.CountOnly); + Assert.True(lastRequest.IncludePrerelease); + Assert.True(lastRequest.IncludeSemVer2); + Assert.Equal("windows azure storage", lastRequest.Query); + Assert.True(lastRequest.LuceneQuery); + Assert.True(lastRequest.ShowDebug); + Assert.Equal("dotnettool", lastRequest.PackageType); + } + + [Theory] + [InlineData("", V2SortBy.Popularity)] + [InlineData(null, V2SortBy.Popularity)] + [InlineData(" ", V2SortBy.Popularity)] + [InlineData("not-real", V2SortBy.Popularity)] + [InlineData("popularity", V2SortBy.Popularity)] + [InlineData("POPULARITY", V2SortBy.Popularity)] + [InlineData(" lastEdited ", V2SortBy.Popularity)] + [InlineData("lastEdited", V2SortBy.LastEditedDesc)] + [InlineData("LASTEDITED", V2SortBy.LastEditedDesc)] + [InlineData("published", V2SortBy.PublishedDesc)] + [InlineData("puBLISHed", V2SortBy.PublishedDesc)] + [InlineData("title-asc", V2SortBy.SortableTitleAsc)] + [InlineData("TITLE-asc", V2SortBy.SortableTitleAsc)] + [InlineData("title-desc", V2SortBy.SortableTitleDesc)] + [InlineData("title-DESC", V2SortBy.SortableTitleDesc)] + [InlineData("CREATED", V2SortBy.Popularity)] + [InlineData("created-asc", V2SortBy.CreatedAsc)] + [InlineData("Created-asc", V2SortBy.CreatedAsc)] + [InlineData("CREATED-desc", V2SortBy.CreatedDesc)] + [InlineData("Created-desc", V2SortBy.CreatedDesc)] + [InlineData("totalDownloads", V2SortBy.Popularity)] + [InlineData("totalDownloads-asc", V2SortBy.TotalDownloadsAsc)] + [InlineData("totalDownloads-desc", V2SortBy.TotalDownloadsDesc)] + [InlineData("TotalDownloads-desc", V2SortBy.TotalDownloadsDesc)] + public async Task ParsesSortBy(string sortBy, V2SortBy expected) + { + await _target.V2SearchAsync(sortBy: sortBy); + + _searchService.Verify( + x => x.V2SearchAsync(It.Is(r => r.SortBy == expected)), + Times.Once); + } + + [Theory] + [MemberData(nameof(SemVerLevels))] + public async Task ParsesSemVerLevel(string semVerLevel, bool includeSemVer2) + { + await _target.V2SearchAsync(semVerLevel: semVerLevel); + + _searchService.Verify( + x => x.V2SearchAsync(It.Is(r => r.IncludeSemVer2 == includeSemVer2)), + Times.Once); + } + } + + public class V3SearchAsync : BaseFacts + { + [Fact] + public async Task InitializesAuxiliaryDataCache() + { + await _target.V3SearchAsync(); + + _auxiliaryDataCache.Verify(x => x.EnsureInitializedAsync(), Times.Once); + } + + [Fact] + public async Task HasDefaultParameters() + { + V3SearchRequest lastRequest = null; + _searchService + .Setup(x => x.V3SearchAsync(It.IsAny())) + .ReturnsAsync(() => _v3SearchResponse) + .Callback(x => lastRequest = x); + + await _target.V3SearchAsync(); + + _searchService.Verify(x => x.V3SearchAsync(It.IsAny()), Times.Once); + Assert.NotNull(lastRequest); + Assert.Equal(0, lastRequest.Skip); + Assert.Equal(20, lastRequest.Take); + Assert.False(lastRequest.IncludePrerelease); + Assert.False(lastRequest.IncludeSemVer2); + Assert.Null(lastRequest.Query); + Assert.Null(lastRequest.PackageType); + Assert.False(lastRequest.ShowDebug); + } + + [Fact] + public async Task SupportsNullParameters() + { + V3SearchRequest lastRequest = null; + _searchService + .Setup(x => x.V3SearchAsync(It.IsAny())) + .ReturnsAsync(() => _v3SearchResponse) + .Callback(x => lastRequest = x); + + await _target.V3SearchAsync( + skip: null, + take: null, + prerelease: null, + semVerLevel: null, + q: null, + packageType: null, + debug: null); + + _searchService.Verify(x => x.V3SearchAsync(It.IsAny()), Times.Once); + Assert.NotNull(lastRequest); + Assert.Equal(0, lastRequest.Skip); + Assert.Equal(20, lastRequest.Take); + Assert.False(lastRequest.IncludePrerelease); + Assert.False(lastRequest.IncludeSemVer2); + Assert.Null(lastRequest.Query); + Assert.Null(lastRequest.PackageType); + Assert.False(lastRequest.ShowDebug); + } + + [Fact] + public async Task UsesProvidedParameters() + { + V3SearchRequest lastRequest = null; + _searchService + .Setup(x => x.V3SearchAsync(It.IsAny())) + .ReturnsAsync(() => _v3SearchResponse) + .Callback(x => lastRequest = x); + + await _target.V3SearchAsync( + skip: -20, + take: 30000, + prerelease: true, + semVerLevel: "2.0.0", + q: "windows azure storage", + packageType: "DotnetTool", + debug: true); + + _searchService.Verify(x => x.V3SearchAsync(It.IsAny()), Times.Once); + Assert.NotNull(lastRequest); + Assert.Equal(-20, lastRequest.Skip); + Assert.Equal(30000, lastRequest.Take); + Assert.True(lastRequest.IncludePrerelease); + Assert.True(lastRequest.IncludeSemVer2); + Assert.Equal("windows azure storage", lastRequest.Query); + Assert.Equal("DotnetTool", lastRequest.PackageType); + Assert.True(lastRequest.ShowDebug); + } + + [Theory] + [MemberData(nameof(SemVerLevels))] + public async Task ParsesSemVerLevel(string semVerLevel, bool includeSemVer2) + { + await _target.V3SearchAsync(semVerLevel: semVerLevel); + + _searchService.Verify( + x => x.V3SearchAsync(It.Is(r => r.IncludeSemVer2 == includeSemVer2)), + Times.Once); + } + } + + public class AutocompleteAsync : BaseFacts + { + [Fact] + public async Task InitializesAuxiliaryDataCache() + { + await _target.AutocompleteAsync(); + + _auxiliaryDataCache.Verify(x => x.EnsureInitializedAsync(), Times.Once); + } + + [Fact] + public async Task HasDefaultParameters() + { + AutocompleteRequest lastRequest = null; + _searchService + .Setup(x => x.AutocompleteAsync(It.IsAny())) + .ReturnsAsync(() => _autocompleteResponse) + .Callback(x => lastRequest = x); + + await _target.AutocompleteAsync(); + + _searchService.Verify(x => x.AutocompleteAsync(It.IsAny()), Times.Once); + Assert.NotNull(lastRequest); + Assert.Equal(0, lastRequest.Skip); + Assert.Equal(20, lastRequest.Take); + Assert.False(lastRequest.IncludePrerelease); + Assert.False(lastRequest.IncludeSemVer2); + Assert.Null(lastRequest.Query); + Assert.False(lastRequest.ShowDebug); + Assert.Null(lastRequest.PackageType); + Assert.Equal(AutocompleteRequestType.PackageIds, lastRequest.Type); + } + + [Fact] + public async Task SupportsNullParameters() + { + AutocompleteRequest lastRequest = null; + _searchService + .Setup(x => x.AutocompleteAsync(It.IsAny())) + .ReturnsAsync(() => _autocompleteResponse) + .Callback(x => lastRequest = x); + + await _target.AutocompleteAsync( + skip: null, + take: null, + prerelease: null, + semVerLevel: null, + q: null, + id: null, + packageType: null, + debug: null); + + _searchService.Verify(x => x.AutocompleteAsync(It.IsAny()), Times.Once); + Assert.NotNull(lastRequest); + Assert.Equal(0, lastRequest.Skip); + Assert.Equal(20, lastRequest.Take); + Assert.False(lastRequest.IncludePrerelease); + Assert.False(lastRequest.IncludeSemVer2); + Assert.Null(lastRequest.Query); + Assert.False(lastRequest.ShowDebug); + Assert.Null(lastRequest.PackageType); + Assert.Equal(AutocompleteRequestType.PackageIds, lastRequest.Type); + } + + [Fact] + public async Task UsesProvidedParameters() + { + AutocompleteRequest lastRequest = null; + _searchService + .Setup(x => x.AutocompleteAsync(It.IsAny())) + .ReturnsAsync(() => _autocompleteResponse) + .Callback(x => lastRequest = x); + + await _target.AutocompleteAsync( + skip: -20, + take: 30000, + prerelease: true, + semVerLevel: "2.0.0", + q: "windows azure storage", + packageType: "DotnetTool", + debug: true); + + _searchService.Verify(x => x.AutocompleteAsync(It.IsAny()), Times.Once); + Assert.NotNull(lastRequest); + Assert.Equal(-20, lastRequest.Skip); + Assert.Equal(30000, lastRequest.Take); + Assert.True(lastRequest.IncludePrerelease); + Assert.True(lastRequest.IncludeSemVer2); + Assert.Equal("windows azure storage", lastRequest.Query); + Assert.True(lastRequest.ShowDebug); + Assert.Equal("DotnetTool", lastRequest.PackageType); + Assert.Equal(AutocompleteRequestType.PackageIds, lastRequest.Type); + } + + [Fact] + public async Task SetsPackageVersionsRequestType() + { + AutocompleteRequest lastRequest = null; + _searchService + .Setup(x => x.AutocompleteAsync(It.IsAny())) + .ReturnsAsync(() => _autocompleteResponse) + .Callback(x => lastRequest = x); + + await _target.AutocompleteAsync( + skip: -20, + take: 30000, + prerelease: true, + semVerLevel: "2.0.0", + id: "windows azure storage", + debug: true); + + _searchService.Verify(x => x.AutocompleteAsync(It.IsAny()), Times.Once); + Assert.NotNull(lastRequest); + Assert.Equal(-20, lastRequest.Skip); + Assert.Equal(30000, lastRequest.Take); + Assert.True(lastRequest.IncludePrerelease); + Assert.True(lastRequest.IncludeSemVer2); + Assert.Equal("windows azure storage", lastRequest.Query); + Assert.True(lastRequest.ShowDebug); + Assert.Equal(AutocompleteRequestType.PackageVersions, lastRequest.Type); + } + + [Fact] + public async Task PrefersPackageIdsRequestType() + { + AutocompleteRequest lastRequest = null; + _searchService + .Setup(x => x.AutocompleteAsync(It.IsAny())) + .ReturnsAsync(() => _autocompleteResponse) + .Callback(x => lastRequest = x); + + await _target.AutocompleteAsync( + skip: -20, + take: 30000, + prerelease: true, + semVerLevel: "2.0.0", + q: "hello world", + id: "windows azure storage", + debug: true); + + _searchService.Verify(x => x.AutocompleteAsync(It.IsAny()), Times.Once); + Assert.NotNull(lastRequest); + Assert.Equal(-20, lastRequest.Skip); + Assert.Equal(30000, lastRequest.Take); + Assert.True(lastRequest.IncludePrerelease); + Assert.True(lastRequest.IncludeSemVer2); + Assert.Equal("hello world", lastRequest.Query); + Assert.True(lastRequest.ShowDebug); + Assert.Equal(AutocompleteRequestType.PackageIds, lastRequest.Type); + } + + [Theory] + [MemberData(nameof(SemVerLevels))] + public async Task ParsesSemVerLevel(string semVerLevel, bool includeSemVer2) + { + await _target.AutocompleteAsync(semVerLevel: semVerLevel); + + _searchService.Verify( + x => x.AutocompleteAsync(It.Is(r => r.IncludeSemVer2 == includeSemVer2)), + Times.Once); + } + } + + public abstract class BaseFacts + { + protected readonly Mock _auxiliaryDataCache; + protected readonly Mock _searchService; + protected readonly Mock _statusService; + protected readonly V2SearchResponse _v2SearchResponse; + protected readonly V3SearchResponse _v3SearchResponse; + protected readonly AutocompleteResponse _autocompleteResponse; + protected readonly SearchController _target; + + public static IEnumerable SemVerLevels => new[] + { + new object[] { null, false }, + new object[] { string.Empty, false }, + new object[] { " ", false }, + new object[] { "something-else", false }, + new object[] { "1", false }, + new object[] { "1.0.0", false }, + new object[] { " 1.0.0 ", false }, + new object[] { "2", true }, + new object[] { "2.0.0", true }, + new object[] { " 2.0.0 ", true }, + new object[] { "3", true }, + new object[] { "3.0.0-beta", true }, + }; + + public BaseFacts() + { + _auxiliaryDataCache = new Mock(); + _searchService = new Mock(); + _statusService = new Mock(); + + _v2SearchResponse = new V2SearchResponse(); + _v3SearchResponse = new V3SearchResponse(); + + _searchService + .Setup(x => x.V2SearchAsync(It.IsAny())) + .ReturnsAsync(() => _v2SearchResponse); + _searchService + .Setup(x => x.V3SearchAsync(It.IsAny())) + .ReturnsAsync(() => _v3SearchResponse); + _searchService + .Setup(x => x.AutocompleteAsync(It.IsAny())) + .ReturnsAsync(() => _autocompleteResponse); + + _target = new SearchController( + _auxiliaryDataCache.Object, + _searchService.Object, + _statusService.Object); + + _target.Request = new HttpRequestMessage(); + _target.Configuration = new HttpConfiguration(); + WebApiConfig.SetSerializerSettings(_target.Configuration.Formatters.JsonFormatter.SerializerSettings); + } + } + } +} diff --git a/tests/NuGet.Services.SearchService.Tests/NuGet.Services.SearchService.Tests.csproj b/tests/NuGet.Services.SearchService.Tests/NuGet.Services.SearchService.Tests.csproj new file mode 100644 index 000000000..58f5c8d6e --- /dev/null +++ b/tests/NuGet.Services.SearchService.Tests/NuGet.Services.SearchService.Tests.csproj @@ -0,0 +1,68 @@ + + + + + Debug + AnyCPU + {F009209D-A663-45E1-87E8-158569A0F097} + Library + Properties + NuGet.Services.SearchService + NuGet.Services.SearchService.Tests + v4.7.2 + 512 + true + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + true + + + + + + + + + + + + 4.10.1 + + + 2.4.1 + + + 2.4.1 + runtime; build; native; contentfiles; analyzers + all + + + + + {1A53FE3D-8041-4773-942F-D73AEF5B82B2} + NuGet.Services.AzureSearch + + + {dd089ab9-6ab3-4aca-8d63-c95a7935b2a7} + NuGet.Services.SearchService + + + + \ No newline at end of file diff --git a/tests/NuGet.Services.SearchService.Tests/Properties/AssemblyInfo.cs b/tests/NuGet.Services.SearchService.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..86ca0b998 --- /dev/null +++ b/tests/NuGet.Services.SearchService.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Services.SearchService.Tests")] +[assembly: ComVisible(false)] +[assembly: Guid("f009209d-a663-45e1-87e8-158569a0f097")] diff --git a/tests/NuGet.Services.V3.Tests/NuGet.Services.V3.Tests.csproj b/tests/NuGet.Services.V3.Tests/NuGet.Services.V3.Tests.csproj new file mode 100644 index 000000000..0d1ab6f60 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/NuGet.Services.V3.Tests.csproj @@ -0,0 +1,100 @@ + + + + + Debug + AnyCPU + {CCB4D5EF-AC84-449D-AC6E-0A0AD295483A} + Library + Properties + NuGet.Services.V3 + NuGet.Services.V3.Tests + v4.7.2 + 512 + true + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 4.10.1 + + + 2.4.1 + + + 2.4.1 + runtime; build; native; contentfiles; analyzers + all + + + + + {e97f23b8-ecb0-4afa-b00c-015c39395fef} + NuGet.Services.Metadata.Catalog + + + {D44C2E89-2D98-44BD-8712-8CCBE4E67C9C} + NuGet.Protocol.Catalog + + + {c3f9a738-9759-4b2b-a50d-6507b28a659b} + NuGet.Services.V3 + + + {1f3bc053-796c-4a35-88f4-955a0f142197} + NuGet.Protocol.Catalog.Tests + + + + + ..\..\build + $(BUILD_SOURCESDIRECTORY)\build + $(NuGetBuildPath) + + + \ No newline at end of file diff --git a/tests/NuGet.Services.V3.Tests/Properties/AssemblyInfo.cs b/tests/NuGet.Services.V3.Tests/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..110538383 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Properties/AssemblyInfo.cs @@ -0,0 +1,9 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Reflection; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("NuGet.Services.V3.Tests")] +[assembly: ComVisible(false)] +[assembly: Guid("ccb4d5ef-ac84-449d-ac6e-0a0ad295483a")] diff --git a/tests/NuGet.Services.V3.Tests/Registration/RegistrationClientFacts.cs b/tests/NuGet.Services.V3.Tests/Registration/RegistrationClientFacts.cs new file mode 100644 index 000000000..e0d081604 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Registration/RegistrationClientFacts.cs @@ -0,0 +1,209 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Moq; +using NuGet.Protocol.Catalog; +using Xunit; +using Xunit.Abstractions; + +namespace NuGet.Protocol.Registration +{ + public class RegistrationClientFacts + { + public class GetIndexAsync : BaseFacts + { + public GetIndexAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task ReturnsNullForNotFound() + { + _simpleHttpClient + .Setup(x => x.DeserializeUrlAsync(It.IsAny())) + .Returns(u => Task.FromResult(new ResponseAndResult( + HttpMethod.Get, + u, + HttpStatusCode.NotFound, + "Not Found", + hasResult: false, + result: null))); + + var result = await _target.GetIndexOrNullAsync(_fakeUrl); + + Assert.Null(result); + } + + [Fact] + public async Task WorksWithInlinedNuGetOrgSnapshot() + { + // Arrange + using (var httpClient = new HttpClient(new TestDataHttpMessageHandler())) + { + var client = GetClient(httpClient); + + // Act + var actual = await client.GetIndexOrNullAsync(TestData.RegistrationIndexInlinedItemsUrl); + + // Assert + Assert.NotNull(actual); + Assert.Equal(1, actual.Count); + Assert.Equal(4, actual.Items.First().Count); + } + } + + [Fact] + public async Task WorksWithNonInlinedNuGetOrgSnapshot() + { + // Arrange + using (var httpClient = new HttpClient(new TestDataHttpMessageHandler())) + { + var client = GetClient(httpClient); + + // Act + var actual = await client.GetIndexOrNullAsync(TestData.RegistrationIndexWithoutInlinedItemsUrl); + + // Assert + Assert.NotNull(actual); + Assert.Equal(19, actual.Count); + Assert.Equal(64, actual.Items.First().Count); + Assert.Equal(55, actual.Items.Last().Count); + } + } + } + + public class GetPageAsync : BaseFacts + { + public GetPageAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task ThrowsForNotFound() + { + _simpleHttpClient + .Setup(x => x.DeserializeUrlAsync(It.IsAny())) + .Returns(u => Task.FromResult(new ResponseAndResult( + HttpMethod.Get, + u, + HttpStatusCode.NotFound, + "Not Found", + hasResult: false, + result: null))); + + await Assert.ThrowsAsync( + () => _target.GetPageAsync(_fakeUrl)); + } + + [Fact] + public async Task WorksWithNuGetOrgSnapshot() + { + // Arrange + using (var httpClient = new HttpClient(new TestDataHttpMessageHandler())) + { + var client = GetClient(httpClient); + + // Act + var actual = await client.GetPageAsync(TestData.RegistrationPageUrl); + + // Assert + Assert.NotNull(actual); + Assert.Equal(55, actual.Count); + } + } + } + + public class GetLeafAsync : BaseFacts + { + public GetLeafAsync(ITestOutputHelper output) : base(output) + { + } + + [Fact] + public async Task ThrowsForNotFound() + { + _simpleHttpClient + .Setup(x => x.DeserializeUrlAsync(It.IsAny())) + .Returns(u => Task.FromResult(new ResponseAndResult( + HttpMethod.Get, + u, + HttpStatusCode.NotFound, + "Not Found", + hasResult: false, + result: null))); + + await Assert.ThrowsAsync( + () => _target.GetLeafAsync(_fakeUrl)); + } + + [Fact] + public async Task WorksWithUnlistedNuGetOrgSnapshot() + { + // Arrange + using (var httpClient = new HttpClient(new TestDataHttpMessageHandler())) + { + var client = GetClient(httpClient); + + // Act + var actual = await client.GetLeafAsync(TestData.RegistrationLeafUnlistedUrl); + + // Assert + Assert.NotNull(actual); + Assert.False(actual.Listed); + Assert.Equal( + "https://api.nuget.org/v3/catalog0/data/2018.11.13.04.43.04/microbuild.core.0.1.1.json", + actual.CatalogEntry); + } + } + + [Fact] + public async Task WorksWithListedNuGetOrgSnapshot() + { + // Arrange + using (var httpClient = new HttpClient(new TestDataHttpMessageHandler())) + { + var client = GetClient(httpClient); + + // Act + var actual = await client.GetLeafAsync(TestData.RegistrationLeafListedUrl); + + // Assert + Assert.NotNull(actual); + Assert.True(actual.Listed); + Assert.Equal( + "https://api.nuget.org/v3/catalog0/data/2018.11.27.18.15.55/newtonsoft.json.12.0.1.json", + actual.CatalogEntry); + } + } + } + + public abstract class BaseFacts + { + protected readonly ITestOutputHelper _output; + protected readonly Mock _simpleHttpClient; + protected readonly string _fakeUrl; + protected readonly RegistrationClient _target; + + public BaseFacts(ITestOutputHelper output) + { + _output = output; + _simpleHttpClient = new Mock(); + + _fakeUrl = "https://example/nuget.versioning/something.json"; + + _target = new RegistrationClient(_simpleHttpClient.Object); + } + + protected RegistrationClient GetClient(HttpClient httpClient) + { + return new RegistrationClient(new SimpleHttpClient( + httpClient, + _output.GetLogger())); + } + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Registration/RegistrationUrlBuilderFacts.cs b/tests/NuGet.Services.V3.Tests/Registration/RegistrationUrlBuilderFacts.cs new file mode 100644 index 000000000..22f68df47 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Registration/RegistrationUrlBuilderFacts.cs @@ -0,0 +1,42 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit; + +namespace NuGet.Protocol.Registration +{ + public class RegistrationUrlBuilderFacts + { + public class GetIndexUrl + { + [Theory] + [InlineData("https://ex/reg", "NuGet.Core", "https://ex/reg/nuget.core/index.json")] + [InlineData("https://ex/reg/", "NuGet.Core", "https://ex/reg/nuget.core/index.json")] + [InlineData("https://ex/reg//", "NuGet.Core", "https://ex/reg/nuget.core/index.json")] + public void ReturnsExpectedUrl(string baseUrl, string id, string expected) + { + var actual = RegistrationUrlBuilder.GetIndexUrl(baseUrl, id); + + Assert.Equal(expected, actual); + } + } + + public class GetLeafUrl + { + [Theory] + [InlineData("https://ex/reg", "NuGet.Core", "3.0.0", "https://ex/reg/nuget.core/3.0.0.json")] + [InlineData("https://ex/reg/", "NuGet.Core", "3.0.0", "https://ex/reg/nuget.core/3.0.0.json")] + [InlineData("https://ex/reg//", "NuGet.Core", "3.0.0", "https://ex/reg/nuget.core/3.0.0.json")] + [InlineData("https://ex/reg/", "NuGet.Core", "3.0.0+git", "https://ex/reg/nuget.core/3.0.0.json")] + [InlineData("https://ex/reg/", "NuGet.Core", "3.0.0.0", "https://ex/reg/nuget.core/3.0.0.json")] + [InlineData("https://ex/reg/", "NuGet.Core", "3.0.0.0-ALPHA", "https://ex/reg/nuget.core/3.0.0-alpha.json")] + [InlineData("https://ex/reg/", "NuGet.Core", "3.0.00-ALPHA.1+foo", "https://ex/reg/nuget.core/3.0.0-alpha.1.json")] + public void ReturnsExpectedUrl(string baseUrl, string id, string version, string expected) + { + var actual = RegistrationUrlBuilder.GetLeafUrl(baseUrl, id, version); + + Assert.Equal(expected, actual); + } + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/Cursor.cs b/tests/NuGet.Services.V3.Tests/Support/Cursor.cs new file mode 100644 index 000000000..e6b6202de --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/Cursor.cs @@ -0,0 +1,14 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using Newtonsoft.Json; + +namespace NuGet.Services +{ + public class Cursor + { + [JsonProperty("value")] + public DateTime Value { get; set; } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/DbSetMockFactory.cs b/tests/NuGet.Services.V3.Tests/Support/DbSetMockFactory.cs new file mode 100644 index 000000000..c81cb446f --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/DbSetMockFactory.cs @@ -0,0 +1,38 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Data.Entity; +using System.Data.Entity.Infrastructure; +using System.Linq; +using Moq; + +namespace NuGet.Services +{ + public static class DbSetMockFactory + { + public static DbSet Create(params T[] sourceList) where T : class + { + var list = new List(sourceList); + + var mockSet = new Mock>(); + mockSet.As>() + .Setup(m => m.GetAsyncEnumerator()) + .Returns(() => new TestDbAsyncEnumerator(list.AsQueryable().GetEnumerator())); + + mockSet.As>() + .Setup(m => m.Provider) + .Returns(() => new TestDbAsyncQueryProvider(list.AsQueryable().Provider)); + + mockSet.As>().Setup(m => m.Expression).Returns(() => list.AsQueryable().Expression); + mockSet.As>().Setup(m => m.ElementType).Returns(() => list.AsQueryable().ElementType); + mockSet.As>().Setup(m => m.GetEnumerator()).Returns(() => list.AsQueryable().GetEnumerator()); + + mockSet.Setup(m => m.Include(It.IsAny())).Returns(() => mockSet.Object); + mockSet.Setup(m => m.Add(It.IsAny())).Callback(e => list.Add(e)); + mockSet.Setup(m => m.Remove(It.IsAny())).Callback(e => list.Remove(e)); + + return mockSet.Object; + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/InMemoryCatalogClient.cs b/tests/NuGet.Services.V3.Tests/Support/InMemoryCatalogClient.cs new file mode 100644 index 000000000..b6e4ded7f --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/InMemoryCatalogClient.cs @@ -0,0 +1,40 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Threading.Tasks; +using NuGet.Protocol.Catalog; + +namespace NuGet.Services +{ + public class InMemoryCatalogClient : ICatalogClient + { + public ConcurrentDictionary PackageDetailsLeaves { get; } = new ConcurrentDictionary(); + + public Task GetIndexAsync(string indexUrl) + { + throw new NotImplementedException(); + } + + public Task GetLeafAsync(string leafUrl) + { + throw new NotImplementedException(); + } + + public Task GetPackageDeleteLeafAsync(string leafUrl) + { + throw new NotImplementedException(); + } + + public Task GetPackageDetailsLeafAsync(string leafUrl) + { + return Task.FromResult(PackageDetailsLeaves[leafUrl]); + } + + public Task GetPageAsync(string pageUrl) + { + throw new NotImplementedException(); + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/InMemoryCloudBlob.cs b/tests/NuGet.Services.V3.Tests/Support/InMemoryCloudBlob.cs new file mode 100644 index 000000000..524dceaa4 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/InMemoryCloudBlob.cs @@ -0,0 +1,231 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using NuGetGallery; + +namespace NuGet.Services +{ + public class InMemoryCloudBlob : ISimpleCloudBlob + { + private static int _nextETag = 0; + private readonly object _lock = new object(); + private string _etag; + + public InMemoryCloudBlob() + { + } + + public InMemoryCloudBlob(string content) + { + Bytes = Encoding.ASCII.GetBytes(content); + Exists = true; + _etag = Interlocked.Increment(ref _nextETag).ToString(); + } + + public BlobProperties Properties { get; } = new CloudBlockBlob(new Uri("https://example/blob")).Properties; + public IDictionary Metadata => throw new NotImplementedException(); + public CopyState CopyState => throw new NotImplementedException(); + public Uri Uri => throw new NotImplementedException(); + public string Name => throw new NotImplementedException(); + public DateTime LastModifiedUtc { get; private set; } = DateTime.UtcNow; + public bool IsSnapshot => throw new NotImplementedException(); + + public string ETag + { + get + { + lock (_lock) + { + return _etag; + } + } + } + + public byte[] Bytes { get; private set; } + public bool Exists { get; private set; } + public string AsString + { + get + { + if (Bytes == null) + { + return null; + } + + return Encoding.UTF8.GetString(Bytes); + } + } + + public Task DeleteIfExistsAsync() + { + lock (_lock) + { + Exists = false; + } + + return Task.CompletedTask; + } + + public Task DownloadToStreamAsync(Stream target) + { + throw new NotImplementedException(); + } + + public Task DownloadToStreamAsync(Stream target, AccessCondition accessCondition) + { + throw new NotImplementedException(); + } + + public Task ExistsAsync() + { + lock (_lock) + { + return Task.FromResult(Exists); + } + } + + public Task FetchAttributesAsync() + { + throw new NotImplementedException(); + } + + public string GetSharedAccessSignature(SharedAccessBlobPermissions permissions, DateTimeOffset? endOfAccess) + { + throw new NotImplementedException(); + } + + public async Task OpenReadAsync(AccessCondition accessCondition) + { + if (accessCondition.IfMatchETag != null || accessCondition.IfNoneMatchETag != null) + { + throw new ArgumentException($"Both {nameof(accessCondition.IfMatchETag)} or {nameof(accessCondition.IfNoneMatchETag)} must be null."); + } + + await Task.Yield(); + + lock (_lock) + { + if (!Exists) + { + throw new StorageException( + new RequestResult + { + HttpStatusCode = (int)HttpStatusCode.NotFound, + }, + "Not found.", + inner: null); + } + + return new MemoryStream(Bytes); + } + } + + public Task OpenReadStreamAsync(TimeSpan serverTimeout, TimeSpan maxExecutionTime, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + public async Task OpenWriteAsync(AccessCondition accessCondition) + { + await Task.Yield(); + + return new RecordingStream(bytes => + { + UploadFromBytes(bytes, accessCondition); + }); + } + + public Task SetMetadataAsync(AccessCondition accessCondition) + { + throw new NotImplementedException(); + } + + public Task SetPropertiesAsync() + { + throw new NotImplementedException(); + } + + public Task SetPropertiesAsync(AccessCondition accessCondition) + { + throw new NotImplementedException(); + } + + public Task SnapshotAsync(CancellationToken token) + { + throw new NotImplementedException(); + } + + public Task StartCopyAsync(ISimpleCloudBlob source, AccessCondition sourceAccessCondition, AccessCondition destAccessCondition) + { + throw new NotImplementedException(); + } + + public Task UploadFromStreamAsync(Stream source, bool overwrite) + { + throw new NotImplementedException(); + } + + public async Task UploadFromStreamAsync(Stream source, AccessCondition accessCondition) + { + if (accessCondition.IfMatchETag != null && accessCondition.IfNoneMatchETag != null) + { + throw new ArgumentException($"Exactly one of {nameof(accessCondition.IfMatchETag)} or {nameof(accessCondition.IfNoneMatchETag)} must be set, not both."); + } + + if (accessCondition.IfNoneMatchETag != null && accessCondition.IfNoneMatchETag != "*") + { + throw new ArgumentException($"{nameof(accessCondition.IfNoneMatchETag)} must be set to either null or '*'."); + } + + await Task.Yield(); + + var buffer = new MemoryStream(); + await source.CopyToAsync(buffer); + var newBytes = buffer.ToArray(); + + UploadFromBytes(newBytes, accessCondition); + } + + private void UploadFromBytes(byte[] newBytes, AccessCondition accessCondition) + { + var newETag = Interlocked.Increment(ref _nextETag).ToString(); + + lock (_lock) + { + if (Exists) + { + if (accessCondition.IfMatchETag != null && accessCondition.IfMatchETag != ETag) + { + throw new InvalidOperationException("The If-Match condition failed because it does not match the current etag."); + } + + if (accessCondition.IfNoneMatchETag == "*") + { + throw new InvalidOperationException("The If-None-Match condition failed because the blob exists."); + } + } + else + { + if (accessCondition.IfMatchETag != null) + { + throw new InvalidOperationException("The If-Match condition failed because the file does not exist."); + } + } + + _etag = newETag; + Bytes = newBytes; + Exists = true; + LastModifiedUtc = DateTime.UtcNow; + } + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/InMemoryCloudBlobClient.cs b/tests/NuGet.Services.V3.Tests/Support/InMemoryCloudBlobClient.cs new file mode 100644 index 000000000..e0706bf5c --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/InMemoryCloudBlobClient.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGetGallery; + +namespace NuGet.Services +{ + public class InMemoryCloudBlobClient : ICloudBlobClient + { + private readonly object _lock = new object(); + + public SortedDictionary Containers { get; } = new SortedDictionary(); + + public ISimpleCloudBlob GetBlobFromUri(Uri uri) + { + throw new NotImplementedException(); + } + + public ICloudBlobContainer GetContainerReference(string containerAddress) + { + lock (_lock) + { + InMemoryCloudBlobContainer container; + if (!Containers.TryGetValue(containerAddress, out container)) + { + container = new InMemoryCloudBlobContainer(); + Containers[containerAddress] = container; + } + + return container; + } + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/InMemoryCloudBlobContainer.cs b/tests/NuGet.Services.V3.Tests/Support/InMemoryCloudBlobContainer.cs new file mode 100644 index 000000000..58a8c0bcb --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/InMemoryCloudBlobContainer.cs @@ -0,0 +1,73 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using NuGetGallery; + +namespace NuGet.Services +{ + public class InMemoryCloudBlobContainer : ICloudBlobContainer + { + private readonly object _lock = new object(); + + public SortedDictionary Blobs { get; } = new SortedDictionary(); + + public Task CreateAsync() + { + throw new NotImplementedException(); + } + + public Task CreateIfNotExistAsync() + { + throw new NotImplementedException(); + } + + public Task DeleteIfExistsAsync() + { + throw new NotImplementedException(); + } + + public Task ExistsAsync(BlobRequestOptions options = null, OperationContext operationContext = null) + { + throw new NotImplementedException(); + } + + public ISimpleCloudBlob GetBlobReference(string blobAddressUri) + { + lock (_lock) + { + InMemoryCloudBlob blob; + if (!Blobs.TryGetValue(blobAddressUri, out blob)) + { + blob = new InMemoryCloudBlob(); + Blobs[blobAddressUri] = blob; + } + + return blob; + } + } + + public Task ListBlobsSegmentedAsync( + string prefix, + bool useFlatBlobListing, + BlobListingDetails blobListingDetails, + int? maxResults, + BlobContinuationToken blobContinuationToken, + BlobRequestOptions options, + OperationContext operationContext, + CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + public Task SetPermissionsAsync(BlobContainerPermissions permissions) + { + throw new NotImplementedException(); + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/InMemoryRegistrationClient.cs b/tests/NuGet.Services.V3.Tests/Support/InMemoryRegistrationClient.cs new file mode 100644 index 000000000..98fa178e5 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/InMemoryRegistrationClient.cs @@ -0,0 +1,36 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Concurrent; +using System.Threading.Tasks; +using NuGet.Protocol.Registration; + +namespace NuGet.Services +{ + public class InMemoryRegistrationClient : IRegistrationClient + { + public ConcurrentDictionary Indexes { get; } = new ConcurrentDictionary(); + public ConcurrentDictionary Pages { get; } = new ConcurrentDictionary(); + public ConcurrentDictionary Leaves { get; } = new ConcurrentDictionary(); + + public Task GetIndexOrNullAsync(string indexUrl) + { + if (Indexes.TryGetValue(indexUrl, out var index)) + { + return Task.FromResult(index); + } + + return Task.FromResult(null); + } + + public Task GetLeafAsync(string leafUrl) + { + return Task.FromResult(Leaves[leafUrl]); + } + + public Task GetPageAsync(string pageUrl) + { + return Task.FromResult(Pages[pageUrl]); + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/IterTools.cs b/tests/NuGet.Services.V3.Tests/Support/IterTools.cs new file mode 100644 index 000000000..f23acd825 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/IterTools.cs @@ -0,0 +1,104 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace NuGet.Services.V3.Support +{ + public static class IterTools + { + /// + /// Source: https://stackoverflow.com/a/3098381 + /// + public static IEnumerable>> CombinationsOfTwoByIndex( + IEnumerable sequenceA, + IEnumerable sequenceBCounts) + { + // This takes as input a sequence of elements (A) and a sequence of element counts (related to another + // sequence B). The count at each position is how many elements from B to combine with that element of + // A. Suppose the input is: + // + // A = [ x, y, z ] + // B = [ 2, 2, 2 ] + // + // The output (in no particular order) would be: + // + // [ + // [ x-1, y-1, z-1 ], [ x-1, y-1, z-2 ], + // [ x-1, y-2, z-1 ], [ x-1, y-2, z-2 ], + // [ x-2, y-1, z-1 ], [ x-2, y-1, z-2 ], + // [ x-2, y-2, z-1 ], [ x-2, y-2, z-2 ], + // ] + // + // This allows the caller to index into sequence B and produce the combinations of A and B. + return from cpLine in CartesianProduct( + from count in sequenceBCounts select Enumerable.Range(1, count)) + select cpLine.Zip(sequenceA, (x1, x2) => Tuple.Create(x2, x1)); + + } + + public static IEnumerable>> CombinationsOfTwo( + IReadOnlyCollection sequenceA, + IReadOnlyList sequenceB) + { + // This has the same behavior as CombinationsOfTwoByIndex but maps the sequence B indexes to actual + // values. Suppose the input is: + // + // A = [ x, y, z ] + // B = [ a, b ] + // + // The output (in no particular order) would be: + // + // [ + // [ x-a, y-a, z-a ], [ x-a, y-a, z-b ], + // [ x-a, y-b, z-a ], [ x-a, y-b, z-b ], + // [ x-b, y-a, z-a ], [ x-b, y-a, z-b ], + // [ x-b, y-b, z-a ], [ x-b, y-b, z-b ], + // ] + // + // This allows the caller to create combinations of A and B where A is fixed but B is varied per + // returned combination. + var arr2 = Enumerable.Repeat(sequenceB.Count, sequenceA.Count); + var combinations = CombinationsOfTwoByIndex(sequenceA, arr2); + return combinations.Select(x => x.Select(t => Tuple.Create(t.Item1, sequenceB[t.Item2 - 1]))); + } + + /// + /// Source: https://stackoverflow.com/a/3098381 + /// + public static IEnumerable> CartesianProduct(IEnumerable> sequences) + { + IEnumerable> emptyProduct = new[] { Enumerable.Empty() }; + return sequences.Aggregate( + emptyProduct, + (accumulator, sequence) => + from accseq in accumulator + from item in sequence + select accseq.Concat(new[] { item }) + ); + } + + /// + /// Source: https://stackoverflow.com/a/999182 + /// + public static IEnumerable> SubsetsOf(IEnumerable source) + { + // This produces all subsets of the input. This includes the input itself and the empty set. The term + // "set" is used to emphasize that order does not matter. The input is assumed to have unique items. If + // it has duplicates, some output sets will also have duplicates. + if (!source.Any()) + { + return Enumerable.Repeat(Enumerable.Empty(), 1); + } + + var element = source.Take(1); + + var haveNots = SubsetsOf(source.Skip(1)); + var haves = haveNots.Select(set => element.Concat(set)); + + return haves.Concat(haveNots); + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/RecordingLogger.cs b/tests/NuGet.Services.V3.Tests/Support/RecordingLogger.cs new file mode 100644 index 000000000..e65233692 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/RecordingLogger.cs @@ -0,0 +1,40 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; + +namespace NuGet.Services +{ + public class RecordingLogger : ILogger + { + private readonly ILogger _inner; + private readonly ConcurrentStack _messages = new ConcurrentStack(); + + public RecordingLogger(ILogger inner) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + } + + public IReadOnlyList Messages => _messages.ToList(); + + public virtual IDisposable BeginScope(TState state) + { + return _inner.BeginScope(state); + } + + public virtual bool IsEnabled(LogLevel logLevel) + { + return _inner.IsEnabled(logLevel); + } + + public virtual void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + _messages.Push(formatter(state, exception)); + _inner.Log(logLevel, eventId, state, exception, formatter); + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/RecordingStream.cs b/tests/NuGet.Services.V3.Tests/Support/RecordingStream.cs new file mode 100644 index 000000000..6cce298f3 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/RecordingStream.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; + +namespace NuGet.Services +{ + public class RecordingStream : MemoryStream + { + private readonly object _lock = new object(); + private Action _onDispose; + + public RecordingStream(Action onDispose) + { + _onDispose = onDispose; + } + + protected override void Dispose(bool disposing) + { + lock (_lock) + { + if (_onDispose != null) + { + _onDispose(ToArray()); + _onDispose = null; + } + } + + base.Dispose(disposing); + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/TestCursorStorage.cs b/tests/NuGet.Services.V3.Tests/Support/TestCursorStorage.cs new file mode 100644 index 000000000..04589234d --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/TestCursorStorage.cs @@ -0,0 +1,65 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json; +using NuGet.Services.Metadata.Catalog.Persistence; + +namespace NuGet.Services +{ + public class TestCursorStorage : Metadata.Catalog.Persistence.Storage + { + public DateTime CursorValue { get; set; } + + public TestCursorStorage(Uri baseAddress) : base(baseAddress) + { + } + + protected override async Task OnLoadAsync( + Uri resourceUri, + CancellationToken cancellationToken) + { + await Task.Yield(); + + return new StringStorageContent(JsonConvert.SerializeObject(new Cursor + { + Value = CursorValue, + })); + } + + protected override Task OnSaveAsync( + Uri resourceUri, + StorageContent content, + CancellationToken cancellationToken) + { + using (var stream = content.GetContentStream()) + using (var reader = new StreamReader(stream)) + { + var json = reader.ReadToEnd(); + var cursor = JsonConvert.DeserializeObject(json); + CursorValue = cursor.Value; + } + + return Task.CompletedTask; + } + + public override bool Exists( + string fileName) => throw new NotImplementedException(); + public override Task> ListAsync( + CancellationToken cancellationToken) => throw new NotImplementedException(); + protected override Task OnCopyAsync( + Uri sourceUri, + IStorage destinationStorage, + Uri destinationUri, + IReadOnlyDictionary destinationProperties, + CancellationToken cancellationToken) => throw new NotImplementedException(); + protected override Task OnDeleteAsync( + Uri resourceUri, + DeleteRequestOptions deleteRequestOptions, + CancellationToken cancellationToken) => throw new NotImplementedException(); + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/TestDbAsyncQueryProvider.cs b/tests/NuGet.Services.V3.Tests/Support/TestDbAsyncQueryProvider.cs new file mode 100644 index 000000000..10518b91e --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/TestDbAsyncQueryProvider.cs @@ -0,0 +1,109 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Collections.Generic; +using System.Data.Entity.Infrastructure; +using System.Linq; +using System.Linq.Expressions; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services +{ + // Copied from https://msdn.microsoft.com/en-us/library/dn314429.aspx + public class TestDbAsyncQueryProvider : IDbAsyncQueryProvider + { + private readonly IQueryProvider _inner; + + public TestDbAsyncQueryProvider(IQueryProvider inner) + { + _inner = inner; + } + + public IQueryable CreateQuery(Expression expression) + { + return new TestDbAsyncEnumerable(expression); + } + + public IQueryable CreateQuery(Expression expression) + { + return new TestDbAsyncEnumerable(expression); + } + + public object Execute(Expression expression) + { + return _inner.Execute(expression); + } + + public TResult Execute(Expression expression) + { + return _inner.Execute(expression); + } + + public Task ExecuteAsync(Expression expression, CancellationToken cancellationToken) + { + return Task.FromResult(Execute(expression)); + } + + public Task ExecuteAsync(Expression expression, CancellationToken cancellationToken) + { + return Task.FromResult(Execute(expression)); + } + } + + public class TestDbAsyncEnumerable : EnumerableQuery, IDbAsyncEnumerable, IQueryable + { + public TestDbAsyncEnumerable(IEnumerable enumerable) + : base(enumerable) + { } + + public TestDbAsyncEnumerable(Expression expression) + : base(expression) + { } + + public IDbAsyncEnumerator GetAsyncEnumerator() + { + return new TestDbAsyncEnumerator(this.AsEnumerable().GetEnumerator()); + } + + IDbAsyncEnumerator IDbAsyncEnumerable.GetAsyncEnumerator() + { + return GetAsyncEnumerator(); + } + + IQueryProvider IQueryable.Provider + { + get { return new TestDbAsyncQueryProvider(this); } + } + } + + public class TestDbAsyncEnumerator : IDbAsyncEnumerator + { + private readonly IEnumerator _inner; + + public TestDbAsyncEnumerator(IEnumerator inner) + { + _inner = inner; + } + + public void Dispose() + { + _inner.Dispose(); + } + + public Task MoveNextAsync(CancellationToken cancellationToken) + { + return Task.FromResult(_inner.MoveNext()); + } + + public T Current + { + get { return _inner.Current; } + } + + object IDbAsyncEnumerator.Current + { + get { return Current; } + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/TestHttpClientHandler.cs b/tests/NuGet.Services.V3.Tests/Support/TestHttpClientHandler.cs new file mode 100644 index 000000000..af995dd17 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/TestHttpClientHandler.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services +{ + public class TestHttpClientHandler : HttpClientHandler + { + public virtual Task OnSendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return await OnSendAsync(request, cancellationToken); + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/TestHttpMessageHandler.cs b/tests/NuGet.Services.V3.Tests/Support/TestHttpMessageHandler.cs new file mode 100644 index 000000000..65abe41f4 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/TestHttpMessageHandler.cs @@ -0,0 +1,23 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace NuGet.Services +{ + public class TestHttpMessageHandler : HttpMessageHandler + { + public virtual Task OnSendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return await OnSendAsync(request, cancellationToken); + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/TestOutputHelperExtensions.cs b/tests/NuGet.Services.V3.Tests/Support/TestOutputHelperExtensions.cs new file mode 100644 index 000000000..ec5bb11dd --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/TestOutputHelperExtensions.cs @@ -0,0 +1,18 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; +using NuGet.Services; + +namespace Xunit.Abstractions +{ + public static class TestOutputHelperExtensions + { + public static RecordingLogger GetLogger(this ITestOutputHelper output) + { + var factory = new LoggerFactory().AddXunit(output); + var inner = factory.CreateLogger(); + return new RecordingLogger(inner); + } + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/V3Data.cs b/tests/NuGet.Services.V3.Tests/Support/V3Data.cs new file mode 100644 index 000000000..e69a8ed97 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/V3Data.cs @@ -0,0 +1,109 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using NuGet.Protocol.Catalog; +using NuGet.Services.Entities; +using NuGet.Versioning; +using NuGetGallery; +using PackageDependency = NuGet.Protocol.Catalog.PackageDependency; + +namespace NuGet.Services +{ + public class V3Data + { + public const string GalleryBaseUrl = "https://example/"; + public const string FlatContainerBaseUrl = "https://example/flat-container/"; + public const string FlatContainerContainerName = "v3-flatcontainer"; + public const string PackageId = "WindowsAzure.Storage"; + public const string FullVersion = "7.1.2-alpha+git"; + public static readonly NuGetVersion ParsedVersion = NuGetVersion.Parse(FullVersion); + public static readonly string NormalizedVersion = ParsedVersion.ToNormalizedString(); + public static readonly string LowerPackageId = PackageId.ToLowerInvariant(); + public static readonly string LowerNormalizedVersion = NormalizedVersion.ToLowerInvariant(); + public static readonly string GalleryLicenseUrl = $"{GalleryBaseUrl}packages/{PackageId}/{NormalizedVersion}/license"; + public static readonly string FlatContainerIconUrl = $"{FlatContainerBaseUrl}{FlatContainerContainerName}/{LowerPackageId}/{LowerNormalizedVersion}/icon"; + public static readonly DateTimeOffset CommitTimestamp = new DateTimeOffset(2018, 12, 13, 12, 30, 0, TimeSpan.Zero); + public static readonly string CommitId = "6b9b24dd-7aec-48ae-afc1-2a117e3d50d1"; + + public static Package PackageEntity => new Package + { + FlattenedAuthors = "Microsoft", + Copyright = "© Microsoft Corporation. All rights reserved.", + Created = new DateTime(2017, 1, 1), + Description = "Description.", + FlattenedDependencies = "Microsoft.Data.OData:5.6.4:net40-client|Newtonsoft.Json:6.0.8:net40-client", + Hash = "oMs9XKzRTsbnIpITcqZ5XAv1h2z6oyJ33+Z/PJx36iVikge/8wm5AORqAv7soKND3v5/0QWW9PQ0ktQuQu9aQQ==", + HashAlgorithm = "SHA512", + IconUrl = "http://go.microsoft.com/fwlink/?LinkID=288890", + IsPrerelease = true, + Language = "en-US", + LastEdited = new DateTime(2017, 1, 2), + LicenseUrl = "http://go.microsoft.com/fwlink/?LinkId=331471", + Listed = true, + MinClientVersion = "2.12", + NormalizedVersion = "7.1.2-alpha", + PackageFileSize = 3039254, + ProjectUrl = "https://github.com/Azure/azure-storage-net", + Published = new DateTime(2017, 1, 3), + ReleaseNotes = "Release notes.", + RequiresLicenseAcceptance = true, + SemVerLevelKey = SemVerLevelKey.SemVer2, + Summary = "Summary.", + Tags = "Microsoft Azure Storage Table Blob File Queue Scalable windowsazureofficial", + Title = "Windows Azure Storage", + Version = "7.1.2.0-alpha+git", + }; + + public static PackageDetailsCatalogLeaf Leaf => new PackageDetailsCatalogLeaf + { + Authors = "Microsoft", + CommitId = CommitId, + CommitTimestamp = CommitTimestamp, + Copyright = "© Microsoft Corporation. All rights reserved.", + Created = new DateTimeOffset(new DateTime(2017, 1, 1), TimeSpan.Zero), + Description = "Description.", + DependencyGroups = new List + { + new PackageDependencyGroup + { + TargetFramework = ".NETFramework4.0-Client", + Dependencies = new List + { + new PackageDependency + { + Id = "Microsoft.Data.OData", + Range = "[5.6.4, )", + }, + new PackageDependency + { + Id = "Newtonsoft.Json", + Range = "[6.0.8, )", + }, + }, + }, + }, + IconUrl = "http://go.microsoft.com/fwlink/?LinkID=288890", + IsPrerelease = true, + Language = "en-US", + LastEdited = new DateTimeOffset(new DateTime(2017, 1, 2), TimeSpan.Zero), + LicenseUrl = "http://go.microsoft.com/fwlink/?LinkId=331471", + Listed = true, + MinClientVersion = "2.12", + PackageHash = "oMs9XKzRTsbnIpITcqZ5XAv1h2z6oyJ33+Z/PJx36iVikge/8wm5AORqAv7soKND3v5/0QWW9PQ0ktQuQu9aQQ==", + PackageHashAlgorithm = "SHA512", + PackageId = PackageId, + PackageSize = 3039254, + PackageVersion = FullVersion, + ProjectUrl = "https://github.com/Azure/azure-storage-net", + Published = new DateTimeOffset(new DateTime(2017, 1, 3), TimeSpan.Zero), + ReleaseNotes = "Release notes.", + RequireLicenseAcceptance = true, + Summary = "Summary.", + Tags = new List { "Microsoft", "Azure", "Storage", "Table", "Blob", "File", "Queue", "Scalable", "windowsazureofficial" }, + Title = "Windows Azure Storage", + VerbatimVersion = "7.1.2.0-alpha+git", + }; + } +} diff --git a/tests/NuGet.Services.V3.Tests/Support/XunitLogger.cs b/tests/NuGet.Services.V3.Tests/Support/XunitLogger.cs new file mode 100644 index 000000000..8647017c2 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/XunitLogger.cs @@ -0,0 +1,61 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Data.Entity; +using System.Linq; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.Logging +{ + public class XunitLogger : ILogger + { + private static readonly char[] NewLineChars = new[] { '\r', '\n' }; + private readonly string _category; + private readonly LogLevel _minLogLevel; + private readonly ITestOutputHelper _output; + + public XunitLogger(ITestOutputHelper output, string category, LogLevel minLogLevel) + { + _minLogLevel = minLogLevel; + _category = category; + _output = output; + } + + public void Log( + LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + if (!IsEnabled(logLevel)) + { + return; + } + var firstLinePrefix = $"| {_category} {logLevel}: "; + var formattedMessage = formatter(state, exception); + if (exception != null) + { + formattedMessage += Environment.NewLine + exception.ToString(); + } + var lines = formattedMessage.Split('\n'); + _output.WriteLine(firstLinePrefix + lines.First().TrimEnd(NewLineChars)); + + var additionalLinePrefix = "|" + new string(' ', firstLinePrefix.Length - 1); + foreach (var line in lines.Skip(1)) + { + _output.WriteLine(additionalLinePrefix + line.TrimEnd(NewLineChars)); + } + } + + public bool IsEnabled(LogLevel logLevel) + => logLevel >= _minLogLevel; + + public IDisposable BeginScope(TState state) + => new NullScope(); + + private class NullScope : IDisposable + { + public void Dispose() + { + } + } + } +} \ No newline at end of file diff --git a/tests/NuGet.Services.V3.Tests/Support/XunitLoggerFactoryExtensions.cs b/tests/NuGet.Services.V3.Tests/Support/XunitLoggerFactoryExtensions.cs new file mode 100644 index 000000000..f63244483 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/XunitLoggerFactoryExtensions.cs @@ -0,0 +1,22 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit.Abstractions; + +namespace Microsoft.Extensions.Logging +{ + public static class XunitLoggerFactoryExtensions + { + public static ILoggerFactory AddXunit(this ILoggerFactory loggerFactory, ITestOutputHelper output) + { + loggerFactory.AddProvider(new XunitLoggerProvider(output)); + return loggerFactory; + } + + public static ILoggerFactory AddXunit(this ILoggerFactory loggerFactory, ITestOutputHelper output, LogLevel minLevel) + { + loggerFactory.AddProvider(new XunitLoggerProvider(output, minLevel)); + return loggerFactory; + } + } +} \ No newline at end of file diff --git a/tests/NuGet.Services.V3.Tests/Support/XunitLoggerProvider.cs b/tests/NuGet.Services.V3.Tests/Support/XunitLoggerProvider.cs new file mode 100644 index 000000000..53525fd48 --- /dev/null +++ b/tests/NuGet.Services.V3.Tests/Support/XunitLoggerProvider.cs @@ -0,0 +1,33 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Xunit.Abstractions; + +namespace Microsoft.Extensions.Logging +{ + public class XunitLoggerProvider : ILoggerProvider + { + private readonly ITestOutputHelper _output; + private readonly LogLevel _minLevel; + + public XunitLoggerProvider(ITestOutputHelper output) + : this(output, LogLevel.Trace) + { + } + + public XunitLoggerProvider(ITestOutputHelper output, LogLevel minLevel) + { + _output = output; + _minLevel = minLevel; + } + + public ILogger CreateLogger(string categoryName) + { + return new XunitLogger(_output, categoryName, _minLevel); + } + + public void Dispose() + { + } + } +} \ No newline at end of file diff --git a/tests/NuGetServicesMetadata.FunctionalTests.sln b/tests/NuGetServicesMetadata.FunctionalTests.sln new file mode 100644 index 000000000..7bb66e5ac --- /dev/null +++ b/tests/NuGetServicesMetadata.FunctionalTests.sln @@ -0,0 +1,36 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.28627.84 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{4AA08EEB-BA47-4ECD-94F3-0700DD50E693}" + ProjectSection(SolutionItems) = preProject + .nuget\packages.config = .nuget\packages.config + EndProjectSection +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BasicSearchTests.FunctionalTests.Core", "BasicSearchTests.FunctionalTests.Core\BasicSearchTests.FunctionalTests.Core.csproj", "{EEA7B6C1-0358-4E67-9D2A-E30B8FF9FF3D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NuGet.Services.AzureSearch.FunctionalTests", "NuGet.Services.AzureSearch.FunctionalTests\NuGet.Services.AzureSearch.FunctionalTests.csproj", "{EAD54C54-E29E-43D5-AE7F-1C194B4EE948}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {EEA7B6C1-0358-4E67-9D2A-E30B8FF9FF3D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EEA7B6C1-0358-4E67-9D2A-E30B8FF9FF3D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EEA7B6C1-0358-4E67-9D2A-E30B8FF9FF3D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EEA7B6C1-0358-4E67-9D2A-E30B8FF9FF3D}.Release|Any CPU.Build.0 = Release|Any CPU + {EAD54C54-E29E-43D5-AE7F-1C194B4EE948}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EAD54C54-E29E-43D5-AE7F-1C194B4EE948}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EAD54C54-E29E-43D5-AE7F-1C194B4EE948}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EAD54C54-E29E-43D5-AE7F-1C194B4EE948}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {C32FA5BC-0AC5-4AE4-938B-66CC1B9AE8CA} + EndGlobalSection +EndGlobal diff --git a/tests/Scripts/DownloadLatestNuGetExeRelease.ps1 b/tests/Scripts/DownloadLatestNuGetExeRelease.ps1 new file mode 100644 index 000000000..b43a98719 --- /dev/null +++ b/tests/Scripts/DownloadLatestNuGetExeRelease.ps1 @@ -0,0 +1,4 @@ +$sourceNugetExe = "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe" +$targetNugetExe = ".\nuget.exe" + +Invoke-WebRequest $sourceNugetExe -OutFile $targetNugetExe \ No newline at end of file diff --git a/tests/Scripts/Import-AzureSearchConfiguration.ps1 b/tests/Scripts/Import-AzureSearchConfiguration.ps1 new file mode 100644 index 000000000..0221bc561 --- /dev/null +++ b/tests/Scripts/Import-AzureSearchConfiguration.ps1 @@ -0,0 +1,42 @@ +[CmdletBinding()] +param ( + [Parameter(Mandatory=$true)][string]$Instance, + [Parameter(Mandatory=$true)][string]$Project, + [Parameter(Mandatory=$true)][string]$PersonalAccessToken, + [Parameter(Mandatory=$true)][string]$Repository, + [Parameter(Mandatory=$true)][string]$Branch = "master", + [Parameter(Mandatory=$true)][string]$ConfigurationName, + [Parameter(Mandatory=$true)][ValidateSet("production", "staging")][string]$Slot = "production" +) + +Write-Host "Importing configuration for Azure Search Functional tests for '$ConfigurationName'" + +Add-Type -AssemblyName System.IO.Compression.FileSystem + +$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f 'PAT', $PersonalAccessToken))) +$headers = @{ Authorization = ("Basic {0}" -f $basicAuth) } + +$filename = "$ConfigurationName.json" +$destinationDirectory = Join-Path $PSScriptRoot "..\NuGet.Services.AzureSearch.FunctionalTests\ExternalConfig" +$destinationPath = Join-Path $destinationDirectory $filename +if (-not (Test-Path $destinationDirectory)) { + New-Item -Path $destinationDirectory -ItemType "directory" +} + +$tempFileName = "temp-$filename" +$tempDestinationPath = Join-Path $destinationDirectory $tempFileName + +Write-Host "Downloading temporary configuration file '$filename' to '$tempDestinationPath'" +$requestUri = "https://$Instance.visualstudio.com/DefaultCollection/$Project/_apis/git/repositories/$Repository/items?api-version=1.0&versionDescriptor.version=$Branch&scopePath=SearchFunctionalConfig\$filename" +$response = Invoke-WebRequest -UseBasicParsing -Uri $requestUri -Headers $headers -OutFile $tempDestinationPath +$configObject = Get-Content -Path $tempDestinationPath | ConvertFrom-Json +Remove-Item -Path $tempDestinationPath + +# Add a field to the file determining which slot should be tested +$configObject | Add-Member -MemberType NoteProperty -Name "Slot" -Value $Slot + +# Save the file and set an environment variable to be used by the functional tests +Write-Host "Writing configuration file with updated values: $destinationPath" +ConvertTo-Json $configObject | Out-File $destinationPath +[Environment]::SetEnvironmentVariable("ConfigurationFilePath", $destinationPath) +Write-Host "##vso[task.setvariable variable=ConfigurationFilePath;]$destinationPath" \ No newline at end of file diff --git a/tests/Scripts/RunAzureSearchFunctionalTests.bat b/tests/Scripts/RunAzureSearchFunctionalTests.bat new file mode 100644 index 000000000..4d3d64d22 --- /dev/null +++ b/tests/Scripts/RunAzureSearchFunctionalTests.bat @@ -0,0 +1,44 @@ +@echo Off + +REM Working directory one level up +cd .. + +REM Configuration +set config=Release +set solutionPath="NuGetServicesMetadata.FunctionalTests.sln" +set exitCode=0 + +REM Required Tools +set msbuild="%PROGRAMFILES(X86)%\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin\msbuild" +set xunit="..\packages\xunit.runner.console.2.1.0\tools\xunit.console.exe" +set nuget="nuget.exe" + +REM Delete old test results +if exist functionaltests.*.xml ( + del functionaltests.*.xml +) + +REM Restore packages +if not exist nuget ( + call PowerShell -NoProfile -ExecutionPolicy Bypass -File %cd%\Scripts\DownloadLatestNuGetExeRelease.ps1 +) + +echo "Restoring all solutions..." +call %nuget% restore "%solutionPath%" -NonInteractive +call %nuget% restore "..\.nuget\packages.config" -PackagesDirectory "..\packages" -NonInteractive +if not "%errorlevel%"=="0" goto failure + +echo "Building solution..." %solutionPath% +REM Build the solution +call %msbuild% "%solutionPath%" /p:Configuration="%config%" /p:Platform="Any CPU" /p:CodeAnalysis=true /m /v:M /fl /flp:LogFile=msbuild.log;Verbosity=diagnostic /nr:false +if not "%errorlevel%"=="0" goto failure + +REM Run functional tests +echo "Running Azure Search functional tests..." +call %xunit% "NuGet.Services.AzureSearch.FunctionalTests\bin\%config%\NuGet.Services.AzureSearch.FunctionalTests.dll" -xml functionaltests.AzureSearchTests.xml +if not "%errorlevel%"=="0" set exitCode=-1 + +exit /B %exitCode% + +:failure +exit /b -1 \ No newline at end of file