Permalink
Browse files

Initial drop. Functionality to extract tweet data from .js files as e…

…xported by Twitter, and push into ElasticSearch. Noddy UI to facilitate the process.
  • Loading branch information...
0 parents commit c2eb0fcbe28db6d78e1aceed12b543d237afc084 @AdaTheDev committed Feb 10, 2013
Showing with 96,988 additions and 0 deletions.
  1. +3 −0 .gitignore
  2. +2 −0 ElasticTweets.Library.UnitTests/.gitignore
  3. +17 −0 ElasticTweets.Library.UnitTests/Common/TestCases.cs
  4. +85 −0 ElasticTweets.Library.UnitTests/ElasticConnectionSettingsFixture.cs
  5. +97 −0 ElasticTweets.Library.UnitTests/ElasticTweets.Library.UnitTests.csproj
  6. +125 −0 ElasticTweets.Library.UnitTests/IO/TweetFileParserFixture.cs
  7. +49 −0 ElasticTweets.Library.UnitTests/ImportResultFixture.cs
  8. +29 −0 ElasticTweets.Library.UnitTests/ImportedFileFixture.cs
  9. +234 −0 ElasticTweets.Library.UnitTests/ImporterFixture.cs
  10. +36 −0 ElasticTweets.Library.UnitTests/Properties/AssemblyInfo.cs
  11. +52 −0 ElasticTweets.Library.UnitTests/Providers/ClientProviderFixture.cs
  12. +94 −0 ElasticTweets.Library.UnitTests/Resources/TestFileContents.Designer.cs
  13. +172 −0 ElasticTweets.Library.UnitTests/Resources/TestFileContents.resx
  14. +7 −0 ElasticTweets.Library.UnitTests/packages.config
  15. +2 −0 ElasticTweets.Library/.gitignore
  16. +36 −0 ElasticTweets.Library/ElasticConnectionSettings.cs
  17. +76 −0 ElasticTweets.Library/ElasticTweets.Library.csproj
  18. +9 −0 ElasticTweets.Library/IElasticConnectionSettings.cs
  19. +27 −0 ElasticTweets.Library/IO/FileSystem.cs
  20. +10 −0 ElasticTweets.Library/IO/IFileSystem.cs
  21. +9 −0 ElasticTweets.Library/IO/ITweetDataFileParser.cs
  22. +56 −0 ElasticTweets.Library/IO/TweetDataFileParser.cs
  23. +26 −0 ElasticTweets.Library/ImportResult.cs
  24. +24 −0 ElasticTweets.Library/ImportedFile.cs
  25. +86 −0 ElasticTweets.Library/Importer.cs
  26. +36 −0 ElasticTweets.Library/Properties/AssemblyInfo.cs
  27. +17 −0 ElasticTweets.Library/Providers/ClientProvider.cs
  28. +9 −0 ElasticTweets.Library/Providers/IClientProvider.cs
  29. +5 −0 ElasticTweets.Library/packages.config
  30. +54 −0 ElasticTweets.sln
  31. +2 −0 ElasticTweets/.gitignore
  32. +93 −0 ElasticTweets/ElasticTweets.csproj
  33. +150 −0 ElasticTweets/Form1.Designer.cs
  34. +27 −0 ElasticTweets/Form1.cs
  35. +120 −0 ElasticTweets/Form1.resx
  36. +21 −0 ElasticTweets/Program.cs
  37. +36 −0 ElasticTweets/Properties/AssemblyInfo.cs
  38. +71 −0 ElasticTweets/Properties/Resources.Designer.cs
  39. +117 −0 ElasticTweets/Properties/Resources.resx
  40. +30 −0 ElasticTweets/Properties/Settings.Designer.cs
  41. +7 −0 ElasticTweets/Properties/Settings.settings
  42. +39 −0 packages/Moq.4.0.10827/License.txt
  43. BIN packages/Moq.4.0.10827/Moq.4.0.10827.nupkg
  44. BIN packages/Moq.4.0.10827/Moq.chm
  45. BIN packages/Moq.4.0.10827/lib/NET35/Moq.dll
  46. BIN packages/Moq.4.0.10827/lib/NET35/Moq.pdb
  47. +5,768 −0 packages/Moq.4.0.10827/lib/NET35/Moq.xml
  48. BIN packages/Moq.4.0.10827/lib/NET40/Moq.dll
  49. BIN packages/Moq.4.0.10827/lib/NET40/Moq.pdb
  50. +5,120 −0 packages/Moq.4.0.10827/lib/NET40/Moq.xml
  51. BIN packages/Moq.4.0.10827/lib/Silverlight4/Castle.Core.dll
  52. BIN packages/Moq.4.0.10827/lib/Silverlight4/Moq.Silverlight.dll
  53. BIN packages/Moq.4.0.10827/lib/Silverlight4/Moq.Silverlight.pdb
  54. +5,101 −0 packages/Moq.4.0.10827/lib/Silverlight4/Moq.Silverlight.xml
  55. BIN packages/NEST.0.9.16.0/NEST.0.9.16.0.nupkg
  56. BIN packages/NEST.0.9.16.0/lib/NET4/Nest.Connection.Thrift.dll
  57. BIN packages/NEST.0.9.16.0/lib/NET4/Nest.Connection.Thrift.pdb
  58. +7,610 −0 packages/NEST.0.9.16.0/lib/NET4/Nest.XML
  59. BIN packages/NEST.0.9.16.0/lib/NET4/Nest.dll
  60. BIN packages/NEST.0.9.16.0/lib/NET4/Nest.pdb
  61. BIN packages/NUnit.2.6.2/NUnit.2.6.2.nupkg
  62. BIN packages/NUnit.2.6.2/lib/nunit.framework.dll
  63. +10,899 −0 packages/NUnit.2.6.2/lib/nunit.framework.xml
  64. +15 −0 packages/NUnit.2.6.2/license.txt
  65. BIN packages/Newtonsoft.Json.4.5.11/Newtonsoft.Json.4.5.11.nupkg
  66. BIN packages/Newtonsoft.Json.4.5.11/lib/net20/Newtonsoft.Json.dll
  67. +8,526 −0 packages/Newtonsoft.Json.4.5.11/lib/net20/Newtonsoft.Json.xml
  68. BIN packages/Newtonsoft.Json.4.5.11/lib/net35/Newtonsoft.Json.dll
  69. +7,662 −0 packages/Newtonsoft.Json.4.5.11/lib/net35/Newtonsoft.Json.xml
  70. BIN packages/Newtonsoft.Json.4.5.11/lib/net40/Newtonsoft.Json.dll
  71. +7,905 −0 packages/Newtonsoft.Json.4.5.11/lib/net40/Newtonsoft.Json.xml
  72. BIN packages/Newtonsoft.Json.4.5.11/lib/portable-net40+sl4+wp7+win8/Newtonsoft.Json.dll
  73. +7,091 −0 packages/Newtonsoft.Json.4.5.11/lib/portable-net40+sl4+wp7+win8/Newtonsoft.Json.xml
  74. BIN packages/Newtonsoft.Json.4.5.11/lib/sl3-wp/Newtonsoft.Json.dll
  75. +7,212 −0 packages/Newtonsoft.Json.4.5.11/lib/sl3-wp/Newtonsoft.Json.xml
  76. BIN packages/Newtonsoft.Json.4.5.11/lib/sl4-windowsphone71/Newtonsoft.Json.dll
  77. +7,212 −0 packages/Newtonsoft.Json.4.5.11/lib/sl4-windowsphone71/Newtonsoft.Json.xml
  78. BIN packages/Newtonsoft.Json.4.5.11/lib/sl4/Newtonsoft.Json.dll
  79. +7,234 −0 packages/Newtonsoft.Json.4.5.11/lib/sl4/Newtonsoft.Json.xml
  80. BIN packages/Newtonsoft.Json.4.5.11/lib/winrt45/Newtonsoft.Json.dll
  81. +7,430 −0 packages/Newtonsoft.Json.4.5.11/lib/winrt45/Newtonsoft.Json.xml
  82. +6 −0 packages/repositories.config
@@ -0,0 +1,3 @@
+_ReSharper*
+*.suo
+*.DotSettings*
@@ -0,0 +1,2 @@
+bin/*
+obj/*
@@ -0,0 +1,17 @@
+using System.Collections.Generic;
+
+namespace ElasticTweets.Library.UnitTests.Common
+{
+ public static class TestCases
+ {
+ public static IEnumerable<string> NullOrWhiteSpaceStrings
+ {
+ get
+ {
+ yield return "";
+ yield return " ";
+ yield return null;
+ }
+ }
+ }
+}
@@ -0,0 +1,85 @@
+using System;
+using ElasticTweets.Library.UnitTests.Common;
+using NUnit.Framework;
+
+// ReSharper disable InconsistentNaming
+namespace ElasticTweets.Library.UnitTests
+{
+ [TestFixture]
+ public class ElasticConnectionSettingsFixture
+ {
+ private const string TestHost = "TestHost";
+ private const string TestIndexName = "TestIndex";
+ private const int TestPort = 123;
+ private ElasticConnectionSettings _settings;
+
+ private void InitialiseSettings()
+ {
+ _settings = new ElasticConnectionSettings(TestHost, TestPort, TestIndexName);
+ }
+ #region Constructor Tests
+ [Test]
+ [TestCaseSource(typeof(TestCases), "NullOrWhiteSpaceStrings")]
+ [ExpectedException(typeof(ArgumentException))]
+ public void Constructor_ThrowsWhenHostNotSupplied(string host)
+ {
+ try
+ {
+// ReSharper disable ObjectCreationAsStatement
+ new ElasticConnectionSettings(host, TestPort, TestIndexName);
+// ReSharper restore ObjectCreationAsStatement
+ }
+ catch (ArgumentException argumentException)
+ {
+ Assert.AreEqual("host", argumentException.ParamName, "Invalid ParamName");
+ throw;
+ }
+ }
+
+ [Test]
+ [TestCaseSource(typeof(TestCases), "NullOrWhiteSpaceStrings")]
+ [ExpectedException(typeof(ArgumentException))]
+ public void Constructor_ThrowsWhenIndexnameNotSupplied(string indexName)
+ {
+ try
+ {
+ // ReSharper disable ObjectCreationAsStatement
+ new ElasticConnectionSettings(TestHost, TestPort, indexName);
+ // ReSharper restore ObjectCreationAsStatement
+ }
+ catch (ArgumentException argumentException)
+ {
+ Assert.AreEqual("indexName", argumentException.ParamName, "Invalid ParamName");
+ throw;
+ }
+ }
+
+ [Test]
+ public void Constructor_SetsHost()
+ {
+ InitialiseSettings();
+
+ Assert.AreEqual(TestHost, _settings.Host);
+ }
+
+ [Test]
+ public void Constructor_SetsPort()
+ {
+ InitialiseSettings();
+
+ Assert.AreEqual(TestPort, _settings.Port);
+ }
+
+
+ [Test]
+ public void Constructor_SetsIndexName()
+ {
+ InitialiseSettings();
+
+ Assert.AreEqual(TestIndexName, _settings.IndexName);
+ }
+ #endregion
+
+ }
+}
+// ReSharper restore InconsistentNaming
@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <PropertyGroup>
+ <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+ <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+ <ProductVersion>8.0.30703</ProductVersion>
+ <SchemaVersion>2.0</SchemaVersion>
+ <ProjectGuid>{5300D2DF-1462-4C4D-B3AF-D7BEE82846F6}</ProjectGuid>
+ <OutputType>Library</OutputType>
+ <AppDesignerFolder>Properties</AppDesignerFolder>
+ <RootNamespace>ElasticTweets.Library.UnitTests</RootNamespace>
+ <AssemblyName>ElasticTweets.Library.UnitTests</AssemblyName>
+ <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
+ <FileAlignment>512</FileAlignment>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+ <DebugSymbols>true</DebugSymbols>
+ <DebugType>full</DebugType>
+ <Optimize>false</Optimize>
+ <OutputPath>bin\Debug\</OutputPath>
+ <DefineConstants>DEBUG;TRACE</DefineConstants>
+ <ErrorReport>prompt</ErrorReport>
+ <WarningLevel>4</WarningLevel>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+ <DebugType>pdbonly</DebugType>
+ <Optimize>true</Optimize>
+ <OutputPath>bin\Release\</OutputPath>
+ <DefineConstants>TRACE</DefineConstants>
+ <ErrorReport>prompt</ErrorReport>
+ <WarningLevel>4</WarningLevel>
+ </PropertyGroup>
+ <ItemGroup>
+ <Reference Include="Moq">
+ <HintPath>..\packages\Moq.4.0.10827\lib\NET40\Moq.dll</HintPath>
+ </Reference>
+ <Reference Include="Nest, Version=0.9.16.0, Culture=neutral, processorArchitecture=MSIL">
+ <HintPath>..\packages\NEST.0.9.16.0\lib\NET4\Nest.dll</HintPath>
+ </Reference>
+ <Reference Include="Nest.Connection.Thrift">
+ <HintPath>..\packages\NEST.0.9.16.0\lib\NET4\Nest.Connection.Thrift.dll</HintPath>
+ </Reference>
+ <Reference Include="Newtonsoft.Json">
+ <HintPath>..\packages\Newtonsoft.Json.4.5.11\lib\net40\Newtonsoft.Json.dll</HintPath>
+ </Reference>
+ <Reference Include="nunit.framework, Version=2.6.2.12296, Culture=neutral, PublicKeyToken=96d09a1eb7f44a77, processorArchitecture=MSIL">
+ <SpecificVersion>False</SpecificVersion>
+ <HintPath>..\packages\NUnit.2.6.2\lib\nunit.framework.dll</HintPath>
+ </Reference>
+ <Reference Include="System" />
+ <Reference Include="System.Core" />
+ <Reference Include="System.Xml.Linq" />
+ <Reference Include="System.Data.DataSetExtensions" />
+ <Reference Include="Microsoft.CSharp" />
+ <Reference Include="System.Data" />
+ <Reference Include="System.Xml" />
+ </ItemGroup>
+ <ItemGroup>
+ <Compile Include="Common\TestCases.cs" />
+ <Compile Include="ElasticConnectionSettingsFixture.cs" />
+ <Compile Include="ImportedFileFixture.cs" />
+ <Compile Include="ImporterFixture.cs" />
+ <Compile Include="ImportResultFixture.cs" />
+ <Compile Include="IO\TweetFileParserFixture.cs" />
+ <Compile Include="Properties\AssemblyInfo.cs" />
+ <Compile Include="Providers\ClientProviderFixture.cs" />
+ <Compile Include="Resources\TestFileContents.Designer.cs">
+ <AutoGen>True</AutoGen>
+ <DesignTime>True</DesignTime>
+ <DependentUpon>TestFileContents.resx</DependentUpon>
+ </Compile>
+ </ItemGroup>
+ <ItemGroup>
+ <ProjectReference Include="..\ElasticTweets.Library\ElasticTweets.Library.csproj">
+ <Project>{6A36AE43-2F4E-44AB-B6A9-E17D763C14B2}</Project>
+ <Name>ElasticTweets.Library</Name>
+ </ProjectReference>
+ </ItemGroup>
+ <ItemGroup>
+ <None Include="packages.config" />
+ </ItemGroup>
+ <ItemGroup />
+ <ItemGroup>
+ <EmbeddedResource Include="Resources\TestFileContents.resx">
+ <Generator>ResXFileCodeGenerator</Generator>
+ <LastGenOutput>TestFileContents.Designer.cs</LastGenOutput>
+ </EmbeddedResource>
+ </ItemGroup>
+ <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+ <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
+ Other similar extension points exist, see Microsoft.Common.targets.
+ <Target Name="BeforeBuild">
+ </Target>
+ <Target Name="AfterBuild">
+ </Target>
+ -->
+</Project>
@@ -0,0 +1,125 @@
+using System;
+using System.IO;
+using System.Linq;
+using ElasticTweets.Library.IO;
+using Moq;
+using NUnit.Framework;
+
+// ReSharper disable InconsistentNaming
+namespace ElasticTweets.Library.UnitTests.IO
+{
+ [TestFixture]
+ public class TweetFileParserFixture
+ {
+ private const string TestFileName = "test.js";
+ private Mock<IFileSystem> _mockedFileSystem;
+ private TweetDataFileParser _parser;
+
+ [SetUp]
+ public void Setup()
+ {
+ _mockedFileSystem = new Mock<IFileSystem>(MockBehavior.Strict);
+ _parser = new TweetDataFileParser(_mockedFileSystem.Object);
+ }
+
+ [Test]
+ [ExpectedException(typeof(FileNotFoundException))]
+ public void GetTweets_ThrowsIfFileDoesNotExist()
+ {
+ _mockedFileSystem.Setup(fs => fs.FileExists(It.IsAny<string>())).Returns(false);
+
+ try
+ {
+ _parser.GetTweets(TestFileName);
+ }
+ catch (FileNotFoundException exception)
+ {
+ Assert.AreEqual(TestFileName,exception.FileName);
+ throw;
+ }
+ }
+
+ [Test]
+ [TestCase("")]
+ [TestCase(" ")]
+ [TestCase("\n")]
+ public void GetTweets_ReturnsEmptyEnumerableWhenFileContentsIsEmptyOrWhitespace(string contents)
+ {
+ _mockedFileSystem.Setup(fs => fs.FileExists(It.IsAny<string>())).Returns(true);
+ _mockedFileSystem.Setup(fs => fs.ReadAllText(It.IsAny<string>())).Returns(contents);
+
+ var tweets = _parser.GetTweets(TestFileName);
+
+ Assert.IsEmpty(tweets);
+ }
+
+ [Test]
+ public void GetTweets_ChecksFileExistsBeforeProceeding()
+ {
+ _mockedFileSystem.Setup(fs => fs.FileExists(It.IsAny<string>())).Returns(false);
+ _mockedFileSystem.Setup(fs => fs.ReadAllText(It.IsAny<string>())).Returns("");
+
+ try
+ {
+ _parser.GetTweets(TestFileName);
+ }
+// ReSharper disable EmptyGeneralCatchClause
+ catch{}
+// ReSharper restore EmptyGeneralCatchClause
+
+ _mockedFileSystem.Verify(fs => fs.FileExists(TestFileName), Times.Once());
+ _mockedFileSystem.Verify(fs => fs.ReadAllText(It.IsAny<string>()), Times.Never());
+ }
+
+ [Test]
+ public void GetTweets_RetrievesFileTextViaFileSystem()
+ {
+ _mockedFileSystem.Setup(fs => fs.FileExists(It.IsAny<string>())).Returns(true);
+ _mockedFileSystem.Setup(fs => fs.ReadAllText(It.IsAny<string>())).Returns("");
+
+ _parser.GetTweets(TestFileName);
+
+ _mockedFileSystem.Verify(fs => fs.ReadAllText(TestFileName), Times.Once());
+ }
+
+ [Test]
+ public void GetTweets_ReturnsEmptyEnumerableWhenFileDoesNotContainMultipleLines()
+ {
+ _mockedFileSystem.Setup(fs => fs.FileExists(It.IsAny<string>())).Returns(true);
+ _mockedFileSystem.Setup(fs => fs.ReadAllText(It.IsAny<string>())).Returns("Line1");
+
+ var tweets = _parser.GetTweets(TestFileName);
+
+ Assert.IsEmpty(tweets);
+ }
+
+ [Test]
+ public void GetTweets_ReturnsMultipleTweets()
+ {
+ _mockedFileSystem.Setup(fs => fs.FileExists(It.IsAny<string>())).Returns(true);
+ _mockedFileSystem.Setup(fs => fs.ReadAllText(It.IsAny<string>())).Returns(Resources.TestFileContents.TwoTweets);
+
+ var tweets = _parser.GetTweets(TestFileName).ToArray();
+
+ Assert.AreEqual(2, tweets.Count());
+ Assert.IsTrue(tweets.Any(t => t.text == "Dummy Tweet 1"), "Tweet 1 not found");
+ Assert.IsTrue(tweets.Any(t => t.text == "Dummy Tweet 2 #Hashtag1"), "Tweet 2 not found");
+ }
+
+ [Test]
+ [ExpectedException(typeof(ArgumentException))]
+ public void GetTweets_ThrowsForNonJsFile()
+ {
+ try
+ {
+ _parser.GetTweets("1.txt");
+ }
+ catch (ArgumentException exception)
+ {
+ Assert.AreEqual("fileName", exception.ParamName);
+ throw;
+ }
+ }
+ }
+}
+// ReSharper restore InconsistentNaming
@@ -0,0 +1,49 @@
+using System;
+using System.Linq;
+using NUnit.Framework;
+
+// ReSharper disable InconsistentNaming
+namespace ElasticTweets.Library.UnitTests
+{
+ [TestFixture]
+ public class ImportResultFixture
+ {
+ [Test]
+ public void Constructor_DefaultsImportedFilesToEmptyArray()
+ {
+ var result = new ImportResult();
+
+ Assert.IsEmpty(result.ImportedFiles);
+ }
+
+ [Test]
+ public void AddImportedFile_AddsFileResult()
+ {
+ var result = new ImportResult();
+ var file = new ImportedFile("1.txt", 123);
+
+ result.AddImportedFile(file);
+
+ Assert.AreEqual(1, result.ImportedFiles.Count(), "Expected 1 file in ImportedFiles" );
+ Assert.That(result.ImportedFiles.Any(f => f == file), "Expected file not found");
+ }
+
+ [Test]
+ [ExpectedException(typeof (ArgumentNullException))]
+ public void AddImportedFile_ThrowsForNullFile()
+ {
+ var result = new ImportResult();
+
+ try
+ {
+ result.AddImportedFile(null);
+ }
+ catch (ArgumentNullException exception)
+ {
+ Assert.AreEqual("file", exception.ParamName);
+ throw;
+ }
+ }
+ }
+}
+// ReSharper restore InconsistentNaming
Oops, something went wrong.

0 comments on commit c2eb0fc

Please sign in to comment.