Skip to content
Browse files

Parse tweet data files into Tweet data structure

  • Loading branch information...
1 parent 4c1ff4d commit 63406cf541085935a7c6a1a043dcffddbed1b9ca @AdaTheDev committed
View
17 ElasticTweets.Library.UnitTests/ImporterFixture.cs
@@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
+using ElasticTweets.Library.Data;
using ElasticTweets.Library.IO;
using ElasticTweets.Library.Providers;
using Moq;
@@ -24,7 +25,7 @@ public class ImporterFixture
private Mock<IClientProvider> _mockedClientProvider;
private Mock<IElasticClient> _mockedClient;
private Mock<ITweetDataFileParser> _mockedFileParser;
- private List<dynamic> _testTweets;
+ private List<Tweet> _testTweets;
private Mock<IBulkResponse> _mockedClientSuccessResponse;
[SetUp]
@@ -40,7 +41,7 @@ public void Setup()
.Returns(new BulkOperationResponseItem[] {new BulkIndexResponseItem()});
_mockedClientSuccessResponse.Setup(r => r.ConnectionStatus).Returns(new ConnectionStatus("Test"));
- _testTweets = new List<dynamic>();
+ _testTweets = new List<Tweet>();
}
public void InitialiseImporter(IBulkResponse mockedBulkResponse = null)
@@ -52,9 +53,9 @@ public void InitialiseImporter(IBulkResponse mockedBulkResponse = null)
_mockedFileParser.Setup(fp => fp.GetTweets(It.IsAny<string>())).Returns(_testTweets);
if (mockedBulkResponse == null)
- _mockedClient.Setup(c => c.IndexMany(It.IsAny<IEnumerable<dynamic>>())).Returns(_mockedClientSuccessResponse.Object);
+ _mockedClient.Setup(c => c.IndexMany(It.IsAny<IEnumerable<Tweet>>())).Returns(_mockedClientSuccessResponse.Object);
else
- _mockedClient.Setup(c => c.IndexMany(It.IsAny<IEnumerable<dynamic>>())).Returns(mockedBulkResponse);
+ _mockedClient.Setup(c => c.IndexMany(It.IsAny<IEnumerable<Tweet>>())).Returns(mockedBulkResponse);
_mockedClient.Setup(c => c.IsValid).Returns(true);
@@ -244,11 +245,11 @@ public void Import_CallsElasticClientToIndexTweets()
InitialiseImporter();
_mockedFileSystem.Setup(fs => fs.GetFiles(TestSourceDirectory, "*.js")).Returns(new[] { "1.js" });
_mockedFileSystem.Setup(fs => fs.ReadAllText(It.IsAny<string>())).Returns("");
- _testTweets.Add(new {id=1});
+ _testTweets.Add(new Tweet{id=1});
_importer.Import();
- _mockedClient.Verify(c => c.IndexMany(It.IsAny<IEnumerable<dynamic>>()), Times.Once());
+ _mockedClient.Verify(c => c.IndexMany(It.IsAny<IEnumerable<Tweet>>()), Times.Once());
}
[Test]
@@ -256,7 +257,7 @@ public void Import_ReturnsCorrectImportResultForValidFile()
{
InitialiseImporter();
SetupFileSystem();
- _testTweets.Add(new { id = 1 });
+ _testTweets.Add(new Tweet{ id = 1 });
var result = _importer.Import();
@@ -277,7 +278,7 @@ public void Import_ReturnsCorrectImportResultWhenElasticClientReturnsError()
failureResponse.Setup(r => r.ConnectionStatus).Returns(new ConnectionStatus(new Exception(TestResponseErrorMessage)));
InitialiseImporter(failureResponse.Object);
SetupFileSystem();
- _testTweets.Add(new { id = 1 });
+ _testTweets.Add(new Tweet{ id = 1 });
var result = _importer.Import();
View
17 ElasticTweets.Library/ElasticTweets.Library.csproj
@@ -49,7 +49,21 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
+ <Compile Include="Data\BoundingBox.cs" />
+ <Compile Include="Data\Contributor.cs" />
+ <Compile Include="Data\Coordinates.cs" />
+ <Compile Include="Data\Entities.cs" />
+ <Compile Include="Data\Hashtag.cs" />
+ <Compile Include="Data\Media.cs" />
+ <Compile Include="Data\Place.cs" />
+ <Compile Include="Data\Size.cs" />
+ <Compile Include="Data\Tweet.cs" />
+ <Compile Include="Data\Url.cs" />
+ <Compile Include="Data\User.cs" />
+ <Compile Include="Data\UserIdentifier.cs" />
+ <Compile Include="Data\UserMention.cs" />
<Compile Include="ElasticConnectionSettings.cs" />
+ <Compile Include="ElasticSearcher.cs" />
<Compile Include="IElasticConnectionSettings.cs" />
<Compile Include="ImportFileResult.cs" />
<Compile Include="Importer.cs" />
@@ -61,10 +75,13 @@
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Providers\ClientProvider.cs" />
<Compile Include="Providers\IClientProvider.cs" />
+ <Compile Include="Queries\IElasticQuery.cs" />
+ <Compile Include="Queries\TweetsContainingLinksQuery.cs" />
</ItemGroup>
<ItemGroup>
<None Include="packages.config" />
</ItemGroup>
+ <ItemGroup />
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
View
3 ElasticTweets.Library/IO/ITweetDataFileParser.cs
@@ -1,9 +1,10 @@
using System.Collections.Generic;
+using ElasticTweets.Library.Data;
namespace ElasticTweets.Library.IO
{
public interface ITweetDataFileParser
{
- IEnumerable<object> GetTweets(string fileName);
+ IEnumerable<Tweet> GetTweets(string fileName);
}
}
View
9 ElasticTweets.Library/IO/TweetDataFileParser.cs
@@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
+using ElasticTweets.Library.Data;
using Newtonsoft.Json;
namespace ElasticTweets.Library.IO
@@ -29,7 +30,7 @@ public TweetDataFileParser(IFileSystem fileSystem)
/// </summary>
/// <param name="fileName">.js tweet file to get tweets from</param>
/// <returns>enumerable of dynamics</returns>
- public IEnumerable<dynamic> GetTweets(string fileName)
+ public IEnumerable<Tweet> GetTweets(string fileName)
{
if (!fileName.ToLowerInvariant().EndsWith(".js"))
throw new ArgumentException("Expected to receive a .js file", "fileName");
@@ -40,17 +41,17 @@ public IEnumerable<dynamic> GetTweets(string fileName)
string fileText = _fileSystem.ReadAllText(fileName);
if (String.IsNullOrWhiteSpace(fileText))
- return Enumerable.Empty<dynamic>();
+ return Enumerable.Empty<Tweet>();
// Remove the first line
int firstNewLineChar = fileText.IndexOf('\n');
if (firstNewLineChar < 0)
- return Enumerable.Empty<dynamic>();
+ return Enumerable.Empty<Tweet>();
fileText = fileText.Remove(0, firstNewLineChar + 1);
// Deserialize all the tweets in the file
- return JsonConvert.DeserializeObject<dynamic>(fileText);
+ return JsonConvert.DeserializeObject<Tweet[]>(fileText);
}
}
}
View
3 ElasticTweets.Library/Importer.cs
@@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
+using ElasticTweets.Library.Data;
using ElasticTweets.Library.IO;
using ElasticTweets.Library.Providers;
using Nest;
@@ -81,7 +82,7 @@ private ImportFileResult ProcessFile(string file, IElasticClient client)
try
{
- IEnumerable<dynamic> tweets = _parser.GetTweets(file).ToArray();
+ Tweet[] tweets = _parser.GetTweets(file).ToArray();
result = tweets.Any() ? BuildFileResultFromClientResponse(file, client.IndexMany(tweets)) : new ImportFileResult(file, tweets.Count());
}

0 comments on commit 63406cf

Please sign in to comment.
Something went wrong with that request. Please try again.