Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
243 lines (206 sloc) 12.1 KB
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Reflection;
using System.Reflection.Metadata;
using System.Reflection.PortableExecutable;
using System.Threading.Tasks;
using Microsoft.Azure.WebJobs;
using Microsoft.Extensions.Logging;
using Microsoft.WindowsAzure.Storage.Blob;
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
using NuGet.Packaging;
using NuGetTypeSearch.Approach3.Indexing.Models;
using NuGetTypeSearch.Bindings;
using NuGetTypeSearch.Bindings.Catalog;
using NuGetTypeSearch.Bindings.Search;
namespace NuGetTypeSearch.Approach3.Indexing
{
public static class PackageIndexerWithCustomBinding
{
private static readonly HttpClient HttpClient = SharedHttpClient.Instance;
private static readonly JsonSerializer JsonSerializer = JsonSerializer.CreateDefault(
new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver()
});
[Disable]
[FunctionName("Approach3-02-PackageIndexerWithCustomBinding")]
public static async Task RunAsync(
[QueueTrigger(Constants.IndexingQueue, Connection = Constants.IndexingQueueConnection)]
PackageOperation packageOperation,
[AzureSearchIndex(
SearchServiceName = Constants.SearchServiceName,
SearchServiceKey = Constants.SearchServiceKey,
IndexName = Constants.SearchServiceIndexName,
IndexDocumentType = typeof(PackageDocument),
IndexAction = IndexAction.MergeOrUpload,
CreateOrUpdateIndex = true)]
IAsyncCollector<PackageDocument> documentAddCollector,
[AzureSearchIndex(
SearchServiceName = Constants.SearchServiceName,
SearchServiceKey = Constants.SearchServiceKey,
IndexName = Constants.SearchServiceIndexName,
IndexDocumentType = typeof(PackageDocument),
IndexAction = IndexAction.Delete,
CreateOrUpdateIndex = true)]
IAsyncCollector<PackageDocument> documentDeleteCollector,
[Blob("index/{Id}.{VersionNormalized}.json", FileAccess.ReadWrite, Connection = Constants.IndexConnection)]
CloudBlockBlob packageBlob,
ILogger log)
{
// Build actions
if (packageOperation.IsAdd())
{
await RunAddPackageAsync(packageOperation, packageBlob, log, Constants.DevCommitToSearchIndex
? documentAddCollector
: DummyAsyncCollector.For<PackageDocument>());
}
else if (packageOperation.IsDelete())
{
await RunDeletePackageAsync(packageOperation, packageBlob, Constants.DevCommitToSearchIndex
? documentDeleteCollector
: DummyAsyncCollector.For<PackageDocument>());
}
}
private static async Task RunAddPackageAsync(PackageOperation packageOperation, CloudBlockBlob packageBlob, ILogger log, IAsyncCollector<PackageDocument> documentAddCollector)
{
var packagesToIndex = new List<PackageDocument>();
log.LogInformation("Downloading package {packageId}@{packageVersionNormalized} for indexing...",
packageOperation.Id, packageOperation.Version);
using (var packageInputStream = await HttpClient.GetStreamAsync(packageOperation.PackageUrl))
using (var packageInputSeekableStream = TemporaryFileStream.Create())
{
await packageInputStream.CopyToAsync(packageInputSeekableStream);
packageInputSeekableStream.Position = 0;
log.LogInformation("Finished downloading package {packageId}@{packageVersionNormalized} for indexing...",
packageOperation.Id, packageOperation.Version);
using (var nugetPackage = new PackageArchiveReader(packageInputSeekableStream))
{
log.LogInformation("Analyzing package {packageId}@{packageVersionNormalized}...",
packageOperation.Id, packageOperation.Version);
// Get some metadata
var nuspecReader = nugetPackage.NuspecReader;
var packageIdentity = nuspecReader.GetIdentity();
var packageSummary = nuspecReader.GetDescription();
if (string.IsNullOrEmpty(packageSummary))
{
packageSummary = nuspecReader.GetSummary();
}
var packageToIndex = new PackageDocument(
packageIdentity.Id,
packageIdentity.Version.ToNormalizedString(),
packageIdentity.Version.OriginalVersion,
nuspecReader.GetTitle(),
packageSummary,
nuspecReader.GetAuthors(),
nuspecReader.GetTags(),
nuspecReader.GetIconUrl(),
nuspecReader.GetLicenseUrl(),
nuspecReader.GetProjectUrl(),
packageOperation.Published,
AuxiliaryNuGetData.GetDownloadCount(packageIdentity.Id),
packageOperation.IsListed,
packageIdentity.Version.IsPrerelease);
var targetFrameworks = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var typeNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var frameworkSpecificGroups = nugetPackage.GetReferenceItems();
foreach (var frameworkSpecificGroup in frameworkSpecificGroups)
{
// Get some metadata
var targetFramework = frameworkSpecificGroup.TargetFramework.GetShortFolderName();
targetFrameworks.Add(targetFramework);
log.LogInformation(
"Collecting information for {packageId}@{packageVersionNormalized} and framework {targetFramework}...",
packageOperation.Id, packageOperation.Version, targetFramework);
// Collect assembly data
foreach (var item in frameworkSpecificGroup.Items)
{
var entry = nugetPackage.GetEntry(item);
var entryName = item;
log.LogInformation(
"Collecting assembly information from {entryName} for {packageId}@{packageVersionNormalized} and framework {targetFramework}...",
entryName, packageOperation.Id, packageOperation.Version, targetFramework);
using (var assemblyStream = entry.Open())
using (var assemblySeekableStream = TemporaryFileStream.Create())
{
await assemblyStream.CopyToAsync(assemblySeekableStream);
assemblySeekableStream.Position = 0;
using (var portableExecutableReader = new PEReader(assemblySeekableStream))
{
var metadataReader = portableExecutableReader.GetMetadataReader();
foreach (var typeDefinition in metadataReader.TypeDefinitions.Select(metadataReader
.GetTypeDefinition))
{
if (!typeDefinition.Attributes.HasFlag(TypeAttributes.Public)) continue;
var typeNamespace = metadataReader.GetString(typeDefinition.Namespace);
var typeName = metadataReader.GetString(typeDefinition.Name);
if (typeName.StartsWith("<") || typeName.StartsWith("__Static") ||
typeName.Contains("c__DisplayClass")) continue;
log.LogDebug(
"{packageId}@{packageVersionNormalized}, framework {targetFramework}, entry {entryName}: adding {namespace}.{type}",
packageOperation.Id, packageOperation.Version, targetFramework, entryName, typeNamespace, typeName);
typeNames.Add($"{typeNamespace}.{typeName}");
}
}
}
log.LogInformation(
"Finished collecting assembly information from {entryName} for {packageId}@{packageVersionNormalized} and framework {targetFramework}.",
entryName, packageOperation.Id, packageOperation.Version, targetFramework);
}
log.LogInformation(
"Finished collecting information for {packageId}@{packageVersionNormalized} and framework {targetFramework}.",
packageOperation.Id, packageOperation.Version, targetFramework);
}
packageToIndex.TargetFrameworks = targetFrameworks.ToHashSet();
packageToIndex.TypeNames = typeNames.ToHashSet();
log.LogInformation("Finished analyzing package {packageId}@{packageVersionNormalized}.",
packageOperation.Id, packageOperation.Version);
// Add to index blob
try
{
log.LogInformation("Start storing index blob for package {packageId}@{packageVersionNormalized}...",
packageOperation.Id, packageOperation.Version);
await packageBlob.DeleteIfExistsAsync();
using (var jsonStream = await packageBlob.OpenWriteAsync())
using (var jsonWriter = new StreamWriter(jsonStream))
{
JsonSerializer.Serialize(jsonWriter, packagesToIndex);
}
log.LogInformation("Finished storing index blob for package {packageId}@{packageVersionNormalized}.",
packageOperation.Id, packageOperation.Version);
}
catch (Exception ex)
{
log.LogError(ex, "Error storing index blob for package {packageId}@{packageVersionNormalized}.",
packageOperation.Id, packageOperation.Version);
}
// Add to index
try
{
log.LogInformation("Start storing index document for package {packageId}@{packageVersionNormalized}...",
packageOperation.Id, packageOperation.Version);
await documentAddCollector.AddAsync(packageToIndex);
log.LogInformation("Finished storing index document for package {packageId}@{packageVersionNormalized}.",
packageOperation.Id, packageOperation.Version);
}
catch (Exception ex)
{
log.LogError(ex, "Error storing index document for package {packageId}@{packageVersionNormalized}.",
packageOperation.Id, packageOperation.Version);
}
}
}
}
private static async Task RunDeletePackageAsync(PackageOperation packageOperation, CloudBlockBlob packageBlob, IAsyncCollector<PackageDocument> documentDeleteCollector)
{
// Delete from index blob
await packageBlob.DeleteIfExistsAsync();
// Add delete from index actions
await documentDeleteCollector.AddAsync(new PackageDocument(packageOperation.Id, packageOperation.Version));
}
}
}
You can’t perform that action at this time.