Postworthy.Tasks.AzureCleanUp

New project added to help keep the azure storage cleaned up for
performance purposes.

Postworthy.Models/Repository/Providers/AzureBlobStorageCache.cs

@@ -187,7 +187,6 @@ public override void Remove(string key, IEnumerable<TYPE> obj)
             {
                 Remove(key, o);
             }
-            Cache.Remove(key, obj);
         }
 
         #region Internal Blob Azure Classes

Postworthy.Models/Twitter/TwitterModel.cs

@@ -40,7 +40,7 @@ public sealed class TwitterModel
 
         public readonly PostworthyUser PrimaryUser = null;
 
-        private string VERSION
+        public static string VERSION
         {
             get { return "_v" + Assembly.GetCallingAssembly().GetName().Version.ToString(); }
         }

Postworthy.Tasks.AzureCleanUp/Postworthy.Tasks.AzureCleanUp.csproj

@@ -0,0 +1,100 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{F2DC0FDE-3BB6-485C-9E17-A6B53023AEB6}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>Postworthy.Tasks.AzureCleanUp</RootNamespace>
+    <AssemblyName>Postworthy.Tasks.AzureCleanUp</AssemblyName>
+    <TargetFrameworkVersion>v4.5.1</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="Microsoft.Data.Edm, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\packages\Microsoft.Data.Edm.5.6.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.Data.OData, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\packages\Microsoft.Data.OData.5.6.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.Data.Services.Client, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\packages\Microsoft.Data.Services.Client.5.6.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.WindowsAzure.Configuration">
+      <HintPath>..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.WindowsAzure.Storage">
+      <HintPath>..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
+    </Reference>
+    <Reference Include="Newtonsoft.Json, Version=4.5.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
+      <HintPath>..\packages\Newtonsoft.Json.5.0.8\lib\net45\Newtonsoft.Json.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.configuration" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Spatial, Version=5.6.2.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\packages\System.Spatial.5.6.2\lib\net40\System.Spatial.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="App.config" />
+    <None Include="packages.config" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Postworthy.Models\Postworthy.Models.csproj">
+      <Project>{1628e67a-0233-4f42-a3f7-3ff4a121f9a8}</Project>
+      <Name>Postworthy.Models</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
+  </Target>
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>

Postworthy.Tasks.AzureCleanUp/Program.cs

@@ -0,0 +1,179 @@
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Blob;
+using Microsoft.WindowsAzure.Storage.RetryPolicies;
+using Postworthy.Models.Repository;
+using Postworthy.Models.Twitter;
+using System;
+using System.Collections.Generic;
+using System.Configuration;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Postworthy.Tasks.AzureCleanUp
+{
+    class Program
+    {
+
+        static void Main(string[] args)
+        {
+            var connectionString = ConfigurationManager.AppSettings["AzureStorageConnectionString"];
+            if (string.IsNullOrEmpty(connectionString))
+                throw new Exception("Config Section 'appSettings' missing AzureStorageConnectionString value!");
+
+            var storageAccount = CloudStorageAccount.Parse(connectionString);
+            var blobClient = storageAccount.CreateCloudBlobClient();
+
+            var deleteTasks = new List<Task>(10000000);
+
+            Action<Task> delete = t =>
+            {
+                lock (deleteTasks)
+                {
+                    deleteTasks.Add(t);
+                }
+                Console.Clear();
+                Console.WriteLine("Deleting " + deleteTasks.Count + " items");
+            };
+
+            var version = TwitterModel.VERSION;
+
+            bool cleanVersion = args.Any(a => a.ToLower() == "version");
+            bool shrink = args.Any(a => a.ToLower().StartsWith("shrink"));
+            int newSize = args.Where(a => a.ToLower().StartsWith("shrink")).Select(x => x == "shrink" ? 500 : int.Parse(x.Replace("shrink", ""))).FirstOrDefault();
+
+            blobClient.ListContainers()
+                //.Take(1) //Short circuit for testing
+                .ToList().AsParallel().ForAll(c =>
+            {
+                var index = c.GetDirectoryReference("Index");
+                foreach (var b in index.ListBlobs().Where(x => x is CloudBlockBlob).Cast<CloudBlockBlob>())
+                {
+                    if (cleanVersion && !b.Name.Contains(version))
+                    {
+                        //Delete Index
+                        var i = c.GetBlockBlobReference(b.Name);
+                        delete(i.DeleteIfExistsAsync());
+
+                        //Delete all Tweets
+                        var d = c.GetDirectoryReference(b.Name.Split('/').Last());
+                        foreach (var t in d.ListBlobs().Where(x => x is CloudBlockBlob).Cast<CloudBlockBlob>())
+                        {
+                            delete(t.DeleteIfExistsAsync());
+                        }
+                    }
+                    
+                    if (shrink && newSize > 0 && b.Name.Contains(version))
+                    {
+                        //Get Storage Index
+                        var i = c.GetBlockBlobReference(b.Name);
+                        var storageIndex =  Newtonsoft.Json.JsonConvert.DeserializeObject<StorageEntityIndex>(DownloadBlob(i));
+
+                        if (storageIndex.EntityKeys.Count > newSize)
+                        {
+                            //Delete extra Tweets
+                            var d = c.GetDirectoryReference(b.Name.Split('/').Last());
+                            foreach (var t in d.ListBlobs().Where(x => x is CloudBlockBlob).Cast<CloudBlockBlob>().OrderByDescending(x => x.Properties.LastModified).Skip(newSize))
+                            {
+                                storageIndex.EntityKeys.Remove(t.Name.Split('/').Last());
+                                delete(t.DeleteIfExistsAsync());
+                            }
+
+                            //Update Storage Index
+                            UploadBlob(i, storageIndex);
+                        }
+                    }
+                }
+            });
+
+            Console.WriteLine("Waiting on all tasks to complete");
+            Task.WaitAll(deleteTasks.ToArray());
+        }
+
+        private static string DownloadBlob(CloudBlockBlob blob)
+        {
+            using (var stream = new MemoryStream())
+            {
+                StreamReader reader;
+                try
+                {
+                    blob.DownloadToStream(stream, options: new BlobRequestOptions()
+                    {
+                        RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(5), 3)
+                    });
+                }
+                catch (StorageException se)
+                {
+                    return "";
+                }
+                try
+                {
+                    stream.Seek(0, 0);
+                    reader = new StreamReader(new GZipStream(stream, CompressionMode.Decompress));
+                    return reader.ReadToEnd();
+                }
+                catch
+                {
+                    stream.Seek(0, 0);
+                    reader = new StreamReader(stream);
+                    return reader.ReadToEnd();
+                }
+            }
+        }
+
+        private static void UploadBlob(CloudBlockBlob blob, RepositoryEntity obj)
+        {
+            using (var streamCompressed = new MemoryStream())
+            {
+                using (var gzip = new GZipStream(streamCompressed, CompressionMode.Compress))
+                {
+                    var data = Encoding.UTF8.GetBytes(Newtonsoft.Json.JsonConvert.SerializeObject(obj));
+                    gzip.Write(data, 0, data.Length);
+                    gzip.Flush();
+                    gzip.Close();
+
+                    using (var streamOut = new MemoryStream(streamCompressed.ToArray()))
+                    {
+                        blob.UploadFromStream(streamOut);
+                    }
+                }
+            }
+        }
+
+        private class StorageEntityIndex : RepositoryEntity
+        {
+            public const string DIRECTORY_KEY = "Index";
+            public string Key { get; set; }
+            public StorageEntityIndex()
+            {
+                EntityKeys = new List<string>();
+            }
+
+            public StorageEntityIndex(string key)
+            {
+                this.Key = key;
+                EntityKeys = new List<string>();
+            }
+
+            public StorageEntityIndex(string key, List<string> EntityKeys)
+            {
+                this.Key = key;
+                this.EntityKeys = EntityKeys;
+            }
+
+            public List<string> EntityKeys { get; set; }
+
+            public override string UniqueKey
+            {
+                get { return Key; }
+            }
+
+            public override bool IsEqual(RepositoryEntity other)
+            {
+                return this.UniqueKey == other.UniqueKey;
+            }
+        }
+    }
+}

Postworthy.Tasks.AzureCleanUp/Properties/AssemblyInfo.cs

@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("Postworthy.Tasks.AzureCleanUp")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("Postworthy.Tasks.AzureCleanUp")]
+[assembly: AssemblyCopyright("Copyright ©  2015")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("b6f9d46d-86e4-47b3-b232-b48e0df6357d")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]

Postworthy.Tasks.AzureCleanUp/packages.config

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="Microsoft.Data.Edm" version="5.6.2" targetFramework="net451" />
+  <package id="Microsoft.Data.OData" version="5.6.2" targetFramework="net451" />
+  <package id="Microsoft.Data.Services.Client" version="5.6.2" targetFramework="net451" />
+  <package id="Microsoft.WindowsAzure.ConfigurationManager" version="1.8.0.0" targetFramework="net451" />
+  <package id="Newtonsoft.Json" version="5.0.8" targetFramework="net451" />
+  <package id="System.Spatial" version="5.6.2" targetFramework="net451" />
+  <package id="WindowsAzure.Storage" version="4.3.0" targetFramework="net451" />
+</packages>

Postworthy.sln

@@ -33,6 +33,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		Performance1.psess = Performance1.psess
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Postworthy.Tasks.AzureCleanUp", "Postworthy.Tasks.AzureCleanUp\Postworthy.Tasks.AzureCleanUp.csproj", "{F2DC0FDE-3BB6-485C-9E17-A6B53023AEB6}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -133,6 +135,16 @@ Global
 		{30B2E02D-48A1-4EC8-AE7C-60D59B17D6AB}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
 		{30B2E02D-48A1-4EC8-AE7C-60D59B17D6AB}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{30B2E02D-48A1-4EC8-AE7C-60D59B17D6AB}.Release|x86.ActiveCfg = Release|Any CPU
+		{F2DC0FDE-3BB6-485C-9E17-A6B53023AEB6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F2DC0FDE-3BB6-485C-9E17-A6B53023AEB6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F2DC0FDE-3BB6-485C-9E17-A6B53023AEB6}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{F2DC0FDE-3BB6-485C-9E17-A6B53023AEB6}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{F2DC0FDE-3BB6-485C-9E17-A6B53023AEB6}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F2DC0FDE-3BB6-485C-9E17-A6B53023AEB6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F2DC0FDE-3BB6-485C-9E17-A6B53023AEB6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F2DC0FDE-3BB6-485C-9E17-A6B53023AEB6}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{F2DC0FDE-3BB6-485C-9E17-A6B53023AEB6}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{F2DC0FDE-3BB6-485C-9E17-A6B53023AEB6}.Release|x86.ActiveCfg = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE