RavenDB-2919 Initial infrastructure for encryption support in RavenFS

Raven.Abstractions/Data/Constants.cs

@@ -7,11 +7,11 @@ public static class Constants
 	{
 		static Constants()
 		{
-			InDatabaseKeyVerificationDocumentContents = new RavenJObject
+			InResourceKeyVerificationDocumentContents = new RavenJObject
 			{
 				{"Text", "The encryption is correct."}
 			};
-			InDatabaseKeyVerificationDocumentContents.EnsureCannotBeChangeAndEnableSnapshotting();
+			InResourceKeyVerificationDocumentContents.EnsureCannotBeChangeAndEnableSnapshotting();
 		}
 
 		public const string RavenClientPrimaryServerUrl = "Raven-Client-Primary-Server-Url";
@@ -84,8 +84,8 @@ static Constants()
 		public const string EncryptionKeyBitsPreferenceSetting = "Raven/Encryption/KeyBitsPreference";
 		public const string EncryptIndexes = "Raven/Encryption/EncryptIndexes";
 
-		public const string InDatabaseKeyVerificationDocumentName = "Raven/Encryption/Verification";
-		public static readonly RavenJObject InDatabaseKeyVerificationDocumentContents;
+		public const string InResourceKeyVerificationDocumentName = "Raven/Encryption/Verification";
+		public static readonly RavenJObject InResourceKeyVerificationDocumentContents;
 
 		public const int DefaultGeneratedEncryptionKeyLength = 256/8;
 		public const int MinimumAcceptableEncryptionKeyLength = 64/8;

Raven.Database/Bundles/Encryption/Plugin/DocumentEncryption.cs

@@ -1,7 +1,6 @@
 using System.ComponentModel.Composition;
 using System.IO;
 using Raven.Bundles.Encryption.Settings;
-using Raven.Database;
 using Raven.Database.Plugins;
 using Raven.Json.Linq;
 
@@ -16,7 +15,7 @@ public class DocumentEncryption : AbstractDocumentCodec
 
 		public override void Initialize()
 		{
-			settings = EncryptionSettingsManager.GetEncryptionSettingsForDatabase(Database);
+			settings = EncryptionSettingsManager.GetEncryptionSettingsForResource(Database);
 		}
 
 		public override void SecondStageInit()

Raven.Database/Bundles/Encryption/Plugin/EncryptionSettingsDeleteTrigger.cs

@@ -11,7 +11,7 @@ public class EncryptionSettingsDeleteTrigger : AbstractDeleteTrigger
 	{
 		public override VetoResult AllowDelete(string key, TransactionInformation transactionInformation)
 		{
-			if (key == Constants.InDatabaseKeyVerificationDocumentName)
+			if (key == Constants.InResourceKeyVerificationDocumentName)
 				return VetoResult.Deny("Cannot delete the encryption verification document.");
 
 			return base.AllowDelete(key, transactionInformation);

Raven.Database/Bundles/Encryption/Plugin/EncryptionSettingsPutTrigger.cs

@@ -18,7 +18,7 @@ public class EncryptionSettingsPutTrigger : AbstractPutTrigger
 	{
 		public override VetoResult AllowPut(string key, RavenJObject document, RavenJObject metadata, TransactionInformation transactionInformation)
 		{
-			if (key == Constants.InDatabaseKeyVerificationDocumentName)
+			if (key == Constants.InResourceKeyVerificationDocumentName)
 			{
 				if (Database == null) // we haven't been initialized yet
 					return VetoResult.Allowed;

Raven.Database/Bundles/Encryption/Plugin/IndexEncryption.cs

@@ -24,7 +24,7 @@ public class IndexEncryption : AbstractIndexCodec
 
 		public override void Initialize(DocumentDatabase database)
 		{
-			settings = EncryptionSettingsManager.GetEncryptionSettingsForDatabase(database);
+			settings = EncryptionSettingsManager.GetEncryptionSettingsForResource(database);
 		}
 
 		public override Stream Encode(string key, Stream dataStream)

Raven.Database/Bundles/Encryption/Settings/EncryptionSettings.cs

@@ -31,7 +31,7 @@ public EncryptionSettings(byte[] encryptionKey, Type symmetricAlgorithmType, boo
 		public static bool DontEncrypt(string key)
 		{
 			return key.StartsWith(Constants.DontEncryptDocumentsStartingWith, StringComparison.OrdinalIgnoreCase)
-				&& key != Constants.InDatabaseKeyVerificationDocumentName;
+				&& key != Constants.InResourceKeyVerificationDocumentName;
 		}
 
 		public byte[] EncryptionKey

Raven.Database/Bundles/Encryption/Settings/EncryptionSettingsManager.cs

@@ -1,35 +1,34 @@
 using System;
-using System.Collections.Generic;
 using System.Configuration;
+using System.IO;
 using System.Linq;
 using System.Security.Cryptography;
-using System.Text;
 using System.Threading;
-using System.Threading.Tasks;
+using Microsoft.VisualBasic;
 using Raven.Abstractions.Data;
-using Raven.Abstractions.Extensions;
 using Raven.Database;
-using Raven.Database.Plugins;
+using Raven.Database.FileSystem;
+using Raven.Database.Server.Abstractions;
 using Raven.Json.Linq;
+using Constants = Raven.Abstractions.Data.Constants;
 
 namespace Raven.Bundles.Encryption.Settings
 {
 	internal static class EncryptionSettingsManager
 	{
 		private static readonly string EncryptionSettingsKeyInExtensionsState = Guid.NewGuid().ToString();
 
-		public static EncryptionSettings GetEncryptionSettingsForDatabase(DocumentDatabase database)
+		public static EncryptionSettings GetEncryptionSettingsForResource(IResourceStore resource)
 		{
-			var result = (EncryptionSettings)database.ExtensionsState.GetOrAdd(EncryptionSettingsKeyInExtensionsState, _ =>
+			var result = (EncryptionSettings)resource.ExtensionsState.GetOrAdd(EncryptionSettingsKeyInExtensionsState, _ =>
 			{
-				var type = GetTypeFromName(database.Configuration.Settings[Constants.AlgorithmTypeSetting]);
-				var key = GetKeyFromBase64(database.Configuration.Settings[Constants.EncryptionKeySetting], database.Configuration.Encryption.EncryptionKeyBitsPreference);
-				var encryptIndexes = GetEncryptIndexesFromString(database.Configuration.Settings[Constants.EncryptIndexes], true);
+				var type = GetTypeFromName(resource.Configuration.Settings[Constants.AlgorithmTypeSetting]);
+				var key = GetKeyFromBase64(resource.Configuration.Settings[Constants.EncryptionKeySetting], resource.Configuration.Encryption.EncryptionKeyBitsPreference);
+				var encryptIndexes = GetEncryptIndexesFromString(resource.Configuration.Settings[Constants.EncryptIndexes], true);
 
-				return new EncryptionSettings(key, type, encryptIndexes, database.Configuration.Encryption.EncryptionKeyBitsPreference);
+				return new EncryptionSettings(key, type, encryptIndexes, resource.Configuration.Encryption.EncryptionKeyBitsPreference);
 			});
 
-
 			return result;
 		}
 
@@ -83,6 +82,48 @@ private static Type GetTypeFromName(string typeName)
 			return result;
 		}
 
+		/// <summary>
+		/// Uses an encrypted document to verify that the encryption key is correct and decodes it to the right value.
+		/// </summary>
+		public static void VerifyEncryptionKey(RavenFileSystem fileSystem, EncryptionSettings settings)
+		{
+			RavenJObject doc = null;
+			try
+			{
+				fileSystem.Storage.Batch(accessor =>
+				{
+					try
+					{
+						doc = accessor.GetConfig(Constants.InResourceKeyVerificationDocumentName);
+					}
+					catch (FileNotFoundException)
+					{
+					}
+				});
+			}
+			catch (CryptographicException e)
+			{
+				throw new ConfigurationErrorsException("The file system is encrypted with a different key and/or algorithm than the ones "
+					+ "currently in the configuration file.", e);
+			}
+
+			if (doc != null)
+			{
+				var ravenJTokenEqualityComparer = new RavenJTokenEqualityComparer();
+				if (!ravenJTokenEqualityComparer.Equals(doc, Constants.InResourceKeyVerificationDocumentContents))
+					throw new ConfigurationErrorsException("The file system is encrypted with a different key and/or algorithm than the ones is currently configured");
+			}
+			else
+			{
+				// This is the first time the database is loaded.
+				if (EncryptedDocumentsExist(database))
+					throw new InvalidOperationException("The database already has existing documents, you cannot start using encryption now.");
+
+				var clonedDoc = (RavenJObject)Constants.InResourceKeyVerificationDocumentContents.CreateSnapshot();
+				fileSystem.Storage.Batch(accessor => accessor.SetConfig(Constants.InResourceKeyVerificationDocumentName, clonedDoc));
+			}
+		}
+
 		/// <summary>
 		/// Uses an encrypted document to verify that the encryption key is correct and decodes it to the right value.
 		/// </summary>
@@ -91,7 +132,7 @@ public static void VerifyEncryptionKey(DocumentDatabase database, EncryptionSett
 			JsonDocument doc;
 			try
 			{
-				doc = database.Documents.Get(Constants.InDatabaseKeyVerificationDocumentName, null);
+				doc = database.Documents.Get(Constants.InResourceKeyVerificationDocumentName, null);
 			}
 			catch (CryptographicException e)
 			{
@@ -102,7 +143,7 @@ public static void VerifyEncryptionKey(DocumentDatabase database, EncryptionSett
 			if (doc != null)
 			{
 				var ravenJTokenEqualityComparer = new RavenJTokenEqualityComparer();
-				if (!ravenJTokenEqualityComparer.Equals(doc.DataAsJson,Constants.InDatabaseKeyVerificationDocumentContents))
+				if (!ravenJTokenEqualityComparer.Equals(doc.DataAsJson, Constants.InResourceKeyVerificationDocumentContents))
 					throw new ConfigurationErrorsException("The database is encrypted with a different key and/or algorithm than the ones "
 						+ "currently in the configuration file.");
 			}
@@ -112,8 +153,8 @@ public static void VerifyEncryptionKey(DocumentDatabase database, EncryptionSett
 				if (EncryptedDocumentsExist(database))
 					throw new InvalidOperationException("The database already has existing documents, you cannot start using encryption now.");
 
-				var clonedDoc = (RavenJObject)Constants.InDatabaseKeyVerificationDocumentContents.CreateSnapshot();
-				database.Documents.Put(Constants.InDatabaseKeyVerificationDocumentName, null, clonedDoc, new RavenJObject(), null);
+				var clonedDoc = (RavenJObject)Constants.InResourceKeyVerificationDocumentContents.CreateSnapshot();
+				database.Documents.Put(Constants.InResourceKeyVerificationDocumentName, null, clonedDoc, new RavenJObject(), null);
 			}
 		}
 

Raven.Database/Counters/CounterStorage.cs

@@ -8,6 +8,7 @@
 using Newtonsoft.Json;
 using Raven.Abstractions;
 using Raven.Abstractions.Counters;
+using Raven.Abstractions.Util;
 using Raven.Database.Config;
 using Raven.Database.Counters.Controllers;
 using Raven.Database.Extensions;
@@ -61,6 +62,8 @@ public CounterStorage(string serverUrl, string storageName, InMemoryRavenConfigu
 
             metricsCounters = new CountersMetricsManager();
 			transportState = recievedTransportState ?? new TransportState();
+			Configuration = configuration;
+			ExtensionsState = new AtomicDictionary<object>();
             Initialize();
 		}
 
@@ -74,8 +77,11 @@ public TransportState TransportState
 		{
 			get { return transportState; }
 		}
+		public AtomicDictionary<object> ExtensionsState { get; private set; }
 
-	    public CounterStorageStats CreateStats()
+		public InMemoryRavenConfiguration Configuration { get; private set; }
+
+		public CounterStorageStats CreateStats()
 	    {
 	        using (var reader = CreateReader())
 	        {

Raven.Database/FileSystem/Bundles/Encryption/Plugin/FileEncryption.cs

@@ -0,0 +1,47 @@
+// -----------------------------------------------------------------------
+//  <copyright file="FileEncryption.cs" company="Hibernating Rhinos LTD">
+//      Copyright (c) Hibernating Rhinos LTD. All rights reserved.
+//  </copyright>
+// -----------------------------------------------------------------------
+using System.ComponentModel.Composition;
+using System.IO;
+using Raven.Bundles.Encryption.Settings;
+using Raven.Database.FileSystem.Plugins;
+using Raven.Json.Linq;
+
+namespace Raven.Database.FileSystem.Bundles.Encryption.Plugin
+{
+	[InheritedExport(typeof(AbstractFileCodec))]
+	[ExportMetadata("Order", 5000)]
+	[ExportMetadata("Bundle", "Encryption")]
+	public class FileEncryption : AbstractFileCodec
+	{
+		private EncryptionSettings settings;
+
+		public override void Initialize()
+		{
+			settings = EncryptionSettingsManager.GetEncryptionSettingsForResource(FileSystem);
+		}
+
+		public override void SecondStageInit()
+		{
+			EncryptionSettingsManager.VerifyEncryptionKey(FileSystem, settings);
+		}
+
+		public override Stream Encode(string key, Stream data, RavenJObject metadata)
+		{
+			if (EncryptionSettings.DontEncrypt(key))
+				return data;
+
+			return settings.Codec.Encode(key, data);
+		}
+
+		public override Stream Decode(string key, Stream encodedDataStream, RavenJObject metadata)
+		{
+			if (EncryptionSettings.DontEncrypt(key))
+				return encodedDataStream;
+
+			return settings.Codec.Decode(key, encodedDataStream);
+		}
+	}
+}

Raven.Database/FileSystem/Plugins/AbstractFileCodec.cs

@@ -0,0 +1,37 @@
+// -----------------------------------------------------------------------
+//  <copyright file="AbstractFileCodec.cs" company="Hibernating Rhinos LTD">
+//      Copyright (c) Hibernating Rhinos LTD. All rights reserved.
+//  </copyright>
+// -----------------------------------------------------------------------
+using System.ComponentModel.Composition;
+using System.IO;
+using Raven.Json.Linq;
+
+namespace Raven.Database.FileSystem.Plugins
+{
+	[InheritedExport]
+	public abstract class AbstractFileCodec : IRequiresFileSystemInitialization
+	{
+		public RavenFileSystem FileSystem { get; set; }
+
+		public virtual void Initialize(RavenFileSystem fileSystem)
+		{
+			FileSystem = fileSystem;
+			Initialize();
+		}
+
+		public virtual void Initialize()
+		{
+
+		}
+
+		public virtual void SecondStageInit()
+		{
+
+		}
+
+		public abstract Stream Encode(string key, Stream data, RavenJObject metadata);
+
+		public abstract Stream Decode(string key, Stream encodedDataStream, RavenJObject metadata);
+	}
+}

Raven.Database/FileSystem/Plugins/AbstractFileSystemIndexCodec.cs

@@ -0,0 +1,24 @@
+// -----------------------------------------------------------------------
+//  <copyright file="AbstractFileSystemIndexCodec.cs" company="Hibernating Rhinos LTD">
+//      Copyright (c) Hibernating Rhinos LTD. All rights reserved.
+//  </copyright>
+// -----------------------------------------------------------------------
+using System.IO;
+
+namespace Raven.Database.FileSystem.Plugins
+{
+	public abstract class AbstractFileSystemIndexCodec : IRequiresFileSystemInitialization
+	{
+		public void Initialize(RavenFileSystem fileSystem)
+		{
+		}
+
+		public void SecondStageInit()
+		{
+		}
+
+		public abstract Stream Encode(string key, Stream dataStream);
+
+		public abstract Stream Decode(string key, Stream dataStream);
+	}
+}

Raven.Database/FileSystem/Plugins/IRequiresFileSystemInitialization.cs

@@ -0,0 +1,14 @@
+// -----------------------------------------------------------------------
+//  <copyright file="IRequiresFileSystemInitialization.cs" company="Hibernating Rhinos LTD">
+//      Copyright (c) Hibernating Rhinos LTD. All rights reserved.
+//  </copyright>
+// -----------------------------------------------------------------------
+namespace Raven.Database.FileSystem.Plugins
+{
+	public interface IRequiresFileSystemInitialization
+	{
+		void Initialize(RavenFileSystem fileSystem);
+
+		void SecondStageInit(); 
+	}
+}