Permalink
Browse files

VBucket support

- VBucketNodeLocator
- keyTransformer, nodeLocator and trascoder providers can now be instantiated by a factory so custom initialization is possible. (this brings a breaking change in the config file: their attributes are now elements)
- ported some hashes from hashkit
- unit tests are missing
  • Loading branch information...
1 parent 7321456 commit ff351fade1ed927c7c421082b14da9dfafc883d6 @enyim committed Jul 17, 2010
View
@@ -4,17 +4,14 @@
<sectionGroup name="enyim.com">
<section name="memcached" type="Enyim.Caching.Configuration.MemcachedClientSection, Enyim.Caching" />
</sectionGroup>
-
- <section name="northscale" type="NorthScale.Store.Configuration.NorthScaleClientSection, NorthScale.Store" />
</configSections>
<!--
Use this section as a template if you're connecting to regular memcached servers.
Note: you must have the enyim.com/memcached section if you're using the parameterless constructor of EnyimMemcachedClient.
-->
<enyim.com>
- <!-- binary protocol needs more reallife testing so if you're not
- comfortable with it, change it back to protocol="Text" -->
+ <!-- you can use protocol="Text" if your memcached is < 1.3 but you should probably upgrade -->
<memcached protocol="Binary">
<servers>
<!-- make sure you use the same ordering of nodes in every configuration you have -->
@@ -24,22 +21,13 @@
<add address="127.0.0.1" port="20008" />
</servers>
<socketPool minPoolSize="10" maxPoolSize="100" connectionTimeout="00:10:00" deadTimeout="00:02:00" />
+
+ <!-- uncomment the section below if your memcached instance requires authentication
+
+ <authentication type="Enyim.Caching.Memcached.PlainTextAuthenticator" zone="AUTHZ" userName="user name" password="password" />
+
+ -->
</memcached>
</enyim.com>
- <!--
- Use this section as a template if you're connecting to NorthScale Mecached Servers.
- Note: you must have the top-level northscale section if you're using the parameterless constructor of NorthScaleClient.
- -->
- <northscale transcoder="Enyim.Caching.Memcached.DefaultTranscoder, Enyim.Caching">
- <!-- bucket is optional and can be specified in the constructor,
- so you can use the same configuration to connect to different buckets in the cluster -->
- <servers bucket="enyim">
- <!-- provide at least 2-3 urls from your cluster -->
- <add uri="http://192.168.2.100:8080/pools/default" />
- <add uri="http://192.168.2.102:8080/pools/default" />
- </servers>
- <socketPool minPoolSize="10" maxPoolSize="100" connectionTimeout="00:00:10" />
- </northscale>
-
</configuration>
@@ -10,7 +10,7 @@ namespace Enyim.Caching.Configuration
/// </summary>
public sealed class AuthenticationElement : ConfigurationElement, IAuthenticationConfiguration
{
- // TODO make this element play nice with the configuratino system (allow saving, etc.)
+ // TODO make this element play nice with the configuration system (allow saving, etc.)
private Dictionary<string, object> parameters = new Dictionary<string, object>();
/// <summary>
@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Configuration;
+
+namespace Enyim.Caching.Configuration
+{
+ internal class ConfigurationElementException : ConfigurationErrorsException
+ {
+ public ConfigurationElementException(string message, ConfigurationElement element) : this(message, null, element) { }
+ public ConfigurationElementException(string message, Exception inner, ConfigurationElement element) : base(message, inner, element.ElementInformation.Source, element.ElementInformation.LineNumber) { }
+ }
+}
+
+#region [ License information ]
+/* ************************************************************
+ *
+ * Copyright (c) 2010 Attila Kiskó, enyim.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * ************************************************************/
+#endregion
@@ -1,14 +1,56 @@
using System;
+using System.Linq;
+using System.Net;
namespace Enyim.Caching.Configuration
{
public static class ConfigurationHelper
{
public static void CheckForInterface(Type type, Type interfaceType)
{
+ if (type == null || interfaceType == null) return;
+
if (Array.IndexOf<Type>(type.GetInterfaces(), interfaceType) == -1)
throw new System.Configuration.ConfigurationErrorsException("The type " + type.AssemblyQualifiedName + " must implement " + interfaceType.AssemblyQualifiedName);
}
+
+ internal static IPEndPoint ResolveToEndPoint(string value)
+ {
+ if (String.IsNullOrEmpty(value))
+ throw new ArgumentNullException("value");
+
+ var parts = value.Split(':');
+ if (parts.Length != 2)
+ throw new ArgumentException("host:port is expected", "value");
+
+ int port;
+ if (!Int32.TryParse(parts[1], out port))
+ throw new ArgumentException("Cannot parse port: " + parts[1], "value");
+
+ return ResolveToEndPoint(parts[0], port);
+ }
+
+ internal static IPEndPoint ResolveToEndPoint(string host, int port)
+ {
+ if (String.IsNullOrEmpty(host))
+ throw new ArgumentNullException("host");
+
+ IPAddress address;
+
+ // parse as an IP address
+ if (!IPAddress.TryParse(host, out address))
+ {
+ // not an ip, resolve from dns
+ // TODO we need to find a way to specify whihc ip should be used when the host has several
+ var entry = System.Net.Dns.GetHostEntry(host);
+ address = entry.AddressList.FirstOrDefault(ip => ip.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork);
+
+ if (address == null)
+ throw new ArgumentException(String.Format("Could not resolve host '{0}'.", host));
+ }
+
+ return new IPEndPoint(address, port);
+ }
}
}
@@ -36,39 +36,7 @@ public int Port
/// </summary>
public System.Net.IPEndPoint EndPoint
{
- get
- {
- if (this.endpoint == null)
- {
- IPAddress address;
-
- if (!IPAddress.TryParse(this.Address, out address))
- {
- IPHostEntry entry = System.Net.Dns.GetHostEntry(this.Address);
- IPAddress[] list = entry.AddressList;
-
- if (list.Length == 0)
- throw new ConfigurationErrorsException(String.Format("Could not resolve host '{0}'.", this.Address));
-
- // get the first IPv4 address from the list (not sure how memcached works against ipv6 addresses whihc are not localhost)
- for (int i = 0; i < list.Length; i++)
- {
- if (list[i].AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork)
- {
- address = list[i];
- break;
- }
- }
-
- if (address == null)
- throw new ConfigurationErrorsException(String.Format("Host '{0}' does not have an IPv4 address.", this.Address));
- }
-
- this.endpoint = new System.Net.IPEndPoint(address, this.Port);
- }
-
- return this.endpoint;
- }
+ get { return this.endpoint ?? (this.endpoint = ConfigurationHelper.ResolveToEndPoint(this.Address, this.Port)); }
}
#region [ T:IPAddressValidator ]
@@ -26,19 +26,19 @@ public interface IMemcachedClientConfiguration
IAuthenticationConfiguration Authentication { get; }
/// <summary>
- /// Gets or sets the type of the <see cref="T:Enyim.Caching.Memcached.IMemcachedKeyTransformer"/> which will be used to convert item keys for Memcached.
+ /// Creates an <see cref="T:Enyim.Caching.Memcached.IMemcachedKeyTransformer"/> instance which will be used to convert item keys for Memcached.
/// </summary>
- Type KeyTransformer { get; set; }
+ IMemcachedKeyTransformer CreateKeyTransformer();
/// <summary>
- /// Gets or sets the type of the <see cref="T:Enyim.Caching.Memcached.IMemcachedNodeLocator"/> which will be used to assign items to Memcached nodes.
+ /// Creates an <see cref="T:Enyim.Caching.Memcached.IMemcachedNodeLocator"/> instance which will be used to assign items to Memcached nodes.
/// </summary>
- Type NodeLocator { get; set; }
+ IMemcachedNodeLocator CreateNodeLocator();
/// <summary>
- /// Gets or sets the type of the <see cref="T:Enyim.Caching.Memcached.ITranscoder"/> which will be used serialzie or deserialize items.
+ /// Creates an <see cref="T:Enyim.Caching.Memcached.ITranscoder"/> instance which will be used to serialize or deserialize items.
/// </summary>
- Type Transcoder { get; set; }
+ ITranscoder CreateTranscoder();
/// <summary>
/// Gets or sets the type of the communication between client and server.
@@ -0,0 +1,54 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Security.Cryptography;
+using System.Net;
+using System.Web.Script.Serialization;
+using System.Collections.ObjectModel;
+using System.Web;
+
+namespace Enyim.Caching.Configuration
+{
+ public interface IVBucketConfiguration
+ {
+ HashAlgorithm CreateHashAlgorithm();
+ IList<IPEndPoint> Servers { get; }
+ IList<VBucket> Buckets { get; }
+ }
+
+ public struct VBucket
+ {
+ private int master;
+ private int[] replicas;
+
+ public VBucket(int master, int[] replicas)
+ {
+ this.master = master;
+ this.replicas = replicas;
+ }
+
+ public int Master { get { return this.master; } }
+ public int[] Replicas { get { return this.replicas; } }
+ }
+}
+
+#region [ License information ]
+/* ************************************************************
+ *
+ * Copyright (c) 2010 Attila Kiskó, enyim.com
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * ************************************************************/
+#endregion
@@ -0,0 +1,86 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Security.Cryptography;
+using System.Net;
+using System.Web.Script.Serialization;
+using Enyim.Caching.Configuration;
+
+namespace Enyim.Caching.Configuration
+{
+ /// <summary>
+ /// Parses a json formatted vbucket config.
+ /// </summary>
+ public class JsonVBucketConfig : IVBucketConfiguration
+ {
+ private Func<HashAlgorithm> factory;
+ private IPEndPoint[] servers;
+ private VBucket[] buckets;
+
+ public JsonVBucketConfig(string json)
+ {
+ var config = new JavaScriptSerializer().Deserialize<_JsonConfig>(json);
+
+ if (config.numReplicas < 0)
+ throw new ArgumentException("Invalid numReplicas: " + config.numReplicas, "json");
+
+ if (hashFactory.TryGetValue(config.hashAlgorithm, out this.factory))
+ throw new ArgumentException("Unknown hash algorithm: " + config.hashAlgorithm, "json");
+
+ this.servers = config.serverList.Select(endpoint => ConfigurationHelper.ResolveToEndPoint(endpoint)).ToArray();
+ this.buckets = config.vBucketMap.Select((bucket, index) =>
+ {
+ if (bucket == null || bucket.Length != config.numReplicas + 1)
+ throw new ArgumentException("Invalid bucket definition at index " + index, "json");
+
+ return new VBucket(bucket[0], bucket.Skip(1).Take(config.numReplicas)/* .Where(v => v > -1) */.ToArray());
+ }).ToArray();
+ }
+
+ #region [ _JsonConfig ]
+
+ private class _JsonConfig
+ {
+ public string hashAlgorithm;
+ public int numReplicas;
+ public string[] serverList;
+ public int[][] vBucketMap;
+ }
+
+ #endregion
+ #region [ IVBucketConfiguration ]
+
+ HashAlgorithm IVBucketConfiguration.CreateHashAlgorithm()
+ {
+ return factory();
+ }
+
+ IList<IPEndPoint> IVBucketConfiguration.Servers
+ {
+ get { return this.servers; }
+ }
+
+ IList<VBucket> IVBucketConfiguration.Buckets
+ {
+ get { return this.buckets; }
+ }
+
+ #endregion
+ #region [ hashFactory ]
+
+ private static readonly Dictionary<string, Func<HashAlgorithm>> hashFactory = new Dictionary<string, Func<HashAlgorithm>>(StringComparer.OrdinalIgnoreCase)
+ {
+ { String.Empty, () => new HashkitOneAtATime() },
+ { "default", () => new HashkitOneAtATime() },
+ { "crc", () => new HashkitCrc32() },
+ { "fnv1_32", () => new Enyim.FNV1() },
+ { "fnv1_64", () => new Enyim.FNV1a() },
+ { "fnv1a_32", () => new Enyim.FNV64() },
+ { "fnv1a_64", () => new Enyim.FNV64a() },
+ { "murmur", () => new HashkitMurmur() }
+ };
+
+ #endregion
+ }
+}
Oops, something went wrong.

0 comments on commit ff351fa

Please sign in to comment.