Merge pull request #22 from datasift/sep2015updates

Added support for ODP ingestion - DEV-3592

DataSift/APIHelpers.cs

@@ -159,6 +159,24 @@ public static dynamic ParseParameters(string endpoint, dynamic parameters)
             return result;
         }
 
+        public static string SerializeToJsonLD(dynamic data)
+        {
+            if (data == null)
+                throw new ArgumentNullException("data", "data parameter cannot be null");
+
+            if (!(data.GetType().IsArray))
+                throw new ArgumentException("data", "data parameter must be an array of objects");
+
+            string result = "";
+
+            foreach(var d in data)
+            {
+                result += JsonConvert.SerializeObject(d);
+            }
+
+            return result;
+        }
+
         public static string GetEnumDescription(dynamic enumerationValue)
         {
             Type type = enumerationValue.GetType();

DataSift/DataSift.csproj

@@ -155,6 +155,8 @@
     <Compile Include="Rest\Account\Identity.cs" />
     <Compile Include="Rest\Account\Limit.cs" />
     <Compile Include="Rest\Account\Token.cs" />
+    <Compile Include="Rest\IIngestAPIRequest.cs" />
+    <Compile Include="Rest\ODP.cs" />
     <Compile Include="Rest\Pylon.cs" />
     <Compile Include="Rest\Historics.cs" />
     <Compile Include="Rest\HistoricsPreview.cs" />
@@ -167,6 +169,7 @@
     <Compile Include="Rest\RestAPIRequest.cs" />
     <Compile Include="Rest\RestAPIResponse.cs" />
     <Compile Include="Rest\Source.cs" />
+    <Compile Include="Rest\TooManyRequestsException.cs" />
     <Compile Include="Streaming\DataSiftStream.cs" />
     <Compile Include="Streaming\IStreamConnection.cs" />
     <Compile Include="Streaming\StreamAPIException.cs" />

DataSift/DataSiftClient.cs

@@ -20,19 +20,25 @@ public class DataSiftClient
         private string _username;
         private string _apikey;
         private string _baseUrl = "https://api.datasift.com/";
+        private string _baseIngestUrl = "https://in.datasift.com/";
         private string _apiVersion = "1.2";
         private GetAPIRequestDelegate _getRequest;
+        private GetIngestRequestDelegate _getIngestRequest;
         private GetStreamConnectionDelegate _getConnection;
         private Historics _historics;
         private HistoricsPreview _historicsPreview;
         private Source _source;
         private Push _push;
         private Pylon _pylon;
+        private ODP _odp;
         private Account _account;
+
         public delegate IRestAPIRequest GetAPIRequestDelegate(string username, string apikey, string baseUrl, string apiVersion);
+        public delegate IIngestAPIRequest GetIngestRequestDelegate(string username, string apikey, string baseUrl, string apiVersion);
         public delegate IStreamConnection GetStreamConnectionDelegate(string url);
 
-        public DataSiftClient(string username, string apikey, GetAPIRequestDelegate requestCreator = null, GetStreamConnectionDelegate connectionCreator = null, string baseUrl = null, string apiVersion = null)
+        public DataSiftClient(string username, string apikey, GetAPIRequestDelegate requestCreator = null, GetStreamConnectionDelegate connectionCreator = null, GetIngestRequestDelegate ingestRequestCreator = null,
+            string baseUrl = null, string apiVersion = null, string baseIngestUrl = null)
         {
             Contract.Requires<ArgumentNullException>(username != null);
             Contract.Requires<ArgumentException>(username.Trim().Length > 0);
@@ -46,6 +52,9 @@ public DataSiftClient(string username, string apikey, GetAPIRequestDelegate requ
             if(!String.IsNullOrEmpty(baseUrl))
                 _baseUrl = baseUrl;
 
+            if (!String.IsNullOrEmpty(baseIngestUrl))
+                _baseIngestUrl = baseIngestUrl;
+
             if (!String.IsNullOrEmpty(apiVersion))
                 _apiVersion = apiVersion;
 
@@ -54,6 +63,11 @@ public DataSiftClient(string username, string apikey, GetAPIRequestDelegate requ
             else
                 _getRequest = requestCreator;
 
+            if (ingestRequestCreator == null)
+                _getIngestRequest = GetIngestRequestDefault;
+            else
+                _getIngestRequest = ingestRequestCreator;
+
             _getConnection = connectionCreator;
         }
 
@@ -64,11 +78,21 @@ private IRestAPIRequest GetRequestDefault(string username, string apikey, string
             return new RestAPIRequest(username, apikey, baseUrl, apiVersion);
         }
 
+        private IIngestAPIRequest GetIngestRequestDefault(string username, string apikey, string baseIngestUrl, string apiVersion)
+        {
+            return new RestAPIRequest(username, apikey, baseIngestUrl, apiVersion);
+        }
+
         internal IRestAPIRequest GetRequest()
         {
             return _getRequest(_username, _apikey, _baseUrl,_apiVersion);
         }
 
+        internal IIngestAPIRequest GetIngestRequest()
+        {
+            return _getIngestRequest(_username, _apikey, _baseIngestUrl, _apiVersion);
+        }
+
         #endregion
 
         #region Properties
@@ -127,6 +151,15 @@ public Account Account
             }
         }
 
+        public ODP ODP
+        {
+            get
+            {
+                if (_odp == null) _odp = new ODP(this);
+                return _odp;
+            }
+        }
+
         #endregion
 
         #region Streaming

DataSift/Rest/IIngestAPIRequest.cs

@@ -0,0 +1,14 @@
+using RestSharp;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace DataSift.Rest
+{
+    public interface IIngestAPIRequest
+    {
+        RestAPIResponse Ingest(string endpoint, dynamic data, Method method = Method.POST);
+    }
+}

DataSift/Rest/ODP.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace DataSift.Rest
+{
+    public class ODP
+    {
+        DataSiftClient _client = null;
+
+        internal ODP(DataSiftClient client)
+        {
+            _client = client;
+        }
+
+        public RestAPIResponse Ingest(string sourceId, dynamic data)
+        {
+            Contract.Requires<ArgumentNullException>(sourceId != null);
+            Contract.Requires<ArgumentException>(sourceId.Trim().Length > 0);
+            Contract.Requires<ArgumentException>((sourceId != null) ? Constants.SOURCE_ID_FORMAT.IsMatch(sourceId) : true, Messages.INVALID_SOURCE_ID);
+
+            if (data == null)
+                throw new ArgumentNullException("data", "data parameter cannot be null");
+
+            if (!(data.GetType().IsArray))
+                throw new ArgumentException("data", "data parameter must be an array of objects");
+
+            return _client.GetIngestRequest().Ingest(sourceId, data);
+        }
+    }
+}

DataSift/Rest/RestAPIRequest.cs

@@ -14,23 +14,34 @@
 
 namespace DataSift.Rest
 {
-    internal class RestAPIRequest : IRestAPIRequest
+    internal class RestAPIRequest : IRestAPIRequest, IIngestAPIRequest
     {
-        private RestClient _client;
+        private string _userAgent;
+        private string _baseUrl;
+        private string _apiVersion;
+        private HttpBasicAuthenticator _auth;
 
         internal RestAPIRequest(string username, string apikey, string baseUrl, string apiVersion)
         {
             ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls;
 
-            _client = new RestClient(baseUrl + "v" + apiVersion);
-            _client.Authenticator = new HttpBasicAuthenticator(username, apikey);
+            _auth = new HttpBasicAuthenticator(username, apikey);
 
             var version = Assembly.GetExecutingAssembly().GetName().Version;
-            _client.UserAgent = "DataSift/v" + apiVersion + " Dotnet/v" + version.ToString();
+            _userAgent = "DataSift/v" + apiVersion + " Dotnet/v" + version.ToString();
+
+            _baseUrl = baseUrl;
+            _apiVersion = apiVersion;
+
         }
 
         public RestAPIResponse Request(string endpoint, dynamic parameters = null, RestSharp.Method method = Method.GET)
         {
+            RestClient client;
+            client = new RestClient(_baseUrl + "v" + _apiVersion);
+            client.Authenticator = _auth;
+            client.UserAgent = _userAgent;
+
             var request = new RestRequest(endpoint, method);
 
             RestAPIResponse result = null;
@@ -54,7 +65,7 @@ public RestAPIResponse Request(string endpoint, dynamic parameters = null, RestS
             }
 
 
-            IRestResponse response = _client.Execute(request);
+            IRestResponse response = client.Execute(request);
 
             if(endpoint == "pull")
             {
@@ -96,6 +107,56 @@ public RestAPIResponse Request(string endpoint, dynamic parameters = null, RestS
             return result;
 
         }
+
+        public RestAPIResponse Ingest(string endpoint, dynamic data, Method method = Method.POST)
+        {
+            RestClient client;
+            client = new RestClient(_baseUrl);
+            client.Authenticator = _auth;
+            client.UserAgent = _userAgent;
+
+            var request = new RestRequest(endpoint, method);
+
+            RestAPIResponse result = null;
+
+            if (data != null)
+            {
+                request.AddParameter("application/json", APIHelpers.SerializeToJsonLD(data), ParameterType.RequestBody);
+            }
+
+            IRestResponse response = client.Execute(request);
+
+            result = new RestAPIResponse() { RateLimit = APIHelpers.ParseRateLimitHeaders(response.Headers), StatusCode = response.StatusCode };
+            result.Data = APIHelpers.DeserializeResponse(response.Content);
+
+            switch ((int)response.StatusCode)
+            {
+                // Ok status codes
+                case 200:
+                case 201:
+                case 202:
+                case 204:
+                    break;
+
+                //Error status codes
+                case 400:
+                case 401:
+                case 403:
+                case 404:
+                case 405:
+                case 409:
+                case 413:
+                case 416:
+                case 500:
+                case 503:
+                    throw new RestAPIException(result, (APIHelpers.HasAttr(result.Data, "error")) ? result.Data.error : "The request failed, please see the Data & StatusCode properties for more details.");
+
+                case 429:
+                    throw new TooManyRequestsException(result, (APIHelpers.HasAttr(result.Data, "error")) ? result.Data.error : "The request failed because you've exceeded your API request limit.");
+            }
+
+            return result;
+        }
     }
 
 }

DataSiftExamples/DataSiftExamples.csproj

@@ -100,6 +100,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="AccountIdentity.cs" />
+    <Compile Include="Ingestion.cs" />
     <Compile Include="Pylon.cs" />
     <Compile Include="Core.cs" />
     <Compile Include="Historics.cs" />

DataSiftExamples/Ingestion.cs

@@ -0,0 +1,29 @@
+using DataSift;
+using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace DataSiftExamples
+{
+    static class Ingestion
+    {
+        internal static void Run(string username, string apikey)
+        {
+            var client = new DataSiftClient(username, apikey);
+
+            Console.WriteLine("Running 'Ingestion' example...");
+
+            var response = client.ODP.Ingest("d44757b030a1499492f539591d86fe3e", new[] {
+                    new { 
+                        title = "Dummy content"
+                    }
+                });
+
+            Console.WriteLine("\nIngestion response: " + JsonConvert.SerializeObject(response.Data));
+
+        }
+    }
+}

DataSiftExamples/Program.cs

@@ -63,6 +63,10 @@ static void WaitForExampleChoice()
                     AccountIdentity.Run(username, apikey);
                     break;
 
+                case ConsoleKey.D8:
+                    Ingestion.Run(username, apikey);
+                    break;
+
                 default:
                     Console.WriteLine("Unknown example, please try again.\n");
                     WaitForExampleChoice();
@@ -80,7 +84,8 @@ static void PrintIntro()
             Console.WriteLine("    4. Historics API endpoints");
             Console.WriteLine("    5. Source API endpoints");
             Console.WriteLine("    6. Pylon API endpoints");
-            Console.WriteLine("    7. Account Identity API endpoints"); 
+            Console.WriteLine("    7. Account Identity API endpoints");
+            Console.WriteLine("    8. Ingest data with ODP"); 
             Console.WriteLine("\nPress a key to continue...\n");
         }
     }

DataSiftTests/DataSiftTests.csproj

@@ -145,6 +145,7 @@
     <Compile Include="Streaming.cs" />
     <Compile Include="Pylon.cs" />
     <Compile Include="Account\Identity.cs" />
+    <Compile Include="ODP.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="Account\AccountAPIResponses.settings">

DataSiftTests/MockRestAPIRequest.cs

@@ -15,7 +15,7 @@
 
 namespace DataSiftTests
 {
-    public class MockRestAPIRequest : IRestAPIRequest
+    public class MockRestAPIRequest : IRestAPIRequest, IIngestAPIRequest
     {
         public RestAPIResponse Request(string endpoint, dynamic parameters = null, Method method = Method.GET)
         {
@@ -462,5 +462,20 @@ public PullAPIResponse PullRequest(IDictionary<string, object> prms)
 
             return result;
         }
+
+        public RestAPIResponse Ingest(string endpoint, dynamic data, Method method = Method.POST)
+        {
+            string response = null;
+            RestAPIResponse result = new RestAPIResponse();
+
+            string body = APIHelpers.SerializeToJsonLD(data);
+
+            response = "{\"accepted\":1, \"total_message_bytes\": 1691 }";
+
+            result.StatusCode = HttpStatusCode.OK;
+            result.Data = APIHelpers.DeserializeResponse(response);
+
+            return result;
+        }
     }
 }