Skip to content
This repository has been archived by the owner on Mar 9, 2021. It is now read-only.

Commit

Permalink
Updates tumblr crawler and datamodel
Browse files Browse the repository at this point in the history 
- Updates the tumblr blog crawler and the hidden tumblr datamodel to
reflect tumblr api changes.
  • Loading branch information
@johanneszab
johanneszab committed Apr 18, 2018
1 parent bf5c79a commit 3ea452d
Show file tree
Hide file tree
Showing 4 changed files with 80 additions and 5 deletions.
Binary file modified src/TumblThree/.vs/TumblThree/v15/Server/sqlite3/storage.ide
View file
Binary file not shown.
4 changes: 2 additions & 2 deletions src/TumblThree/SharedAssemblyInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@

[assembly: ComVisible(false)]
[assembly: NeutralResourcesLanguage("en-US", UltimateResourceFallbackLocation.MainAssembly)]
[assembly: AssemblyVersion("1.0.8.47")]
[assembly: AssemblyFileVersion("1.0.8.47")]
[assembly: AssemblyVersion("1.0.8.48")]
[assembly: AssemblyFileVersion("1.0.8.48")]
6 changes: 6 additions & 0 deletions src/TumblThree/TumblThree.Applications/Crawler/TumblrBlogCrawler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,12 @@ private async Task UpdateMetaInformation()

public new T ConvertJsonToClass<T>(string json) where T : new()
{
if (json.Contains("tumblr_api_read"))
{
int jsonStart = json.IndexOf("{");
json = json.Substring(jsonStart);
json = json.Remove(json.Length - 2);
}
try
{
using (MemoryStream ms = new MemoryStream(Encoding.Unicode.GetBytes(json)))
Expand Down
75 changes: 72 additions & 3 deletions src/TumblThree/TumblThree.Applications/DataModels/TumblrSvcJson.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,15 @@ public class Trail
public bool? is_root_item { get; set; }
}

[DataContract]
public class LinkImageDimensions
{
[DataMember(EmitDefaultValue = false)]
public int width { get; set; }
[DataMember(EmitDefaultValue = false)]
public int height { get; set; }
}

[DataContract]
public class OriginalSize
{
Expand Down Expand Up @@ -341,6 +350,17 @@ public class Dialogue
public string phrase { get; set; }
}

[DataContract]
public class PinterestShareWindowClass
{
[DataMember(EmitDefaultValue = false)]
public string url { get; set; }
[DataMember(EmitDefaultValue = false)]
public string name { get; set; }
[DataMember(EmitDefaultValue = false)]
public string dimensions { get; set; }
}

[DataContract]
public class PhotosetPhoto
{
Expand Down Expand Up @@ -418,6 +438,8 @@ public class Post : ICloneable
[DataMember(EmitDefaultValue = false)]
public bool html5_capable { get; set; }
[DataMember(EmitDefaultValue = false)]
public Video video { get; set; }
[DataMember(EmitDefaultValue = false)]
public string thumbnail_url { get; set; }
[DataMember(EmitDefaultValue = false)]
public int thumbnail_width { get; set; }
Expand All @@ -426,7 +448,7 @@ public class Post : ICloneable
[DataMember(EmitDefaultValue = false)]
public float duration { get; set; }
[DataMember(EmitDefaultValue = false)]
public List<Player> player { get; set; }
public object player { get; set; }
[DataMember(EmitDefaultValue = false)]
public string audio_url { get; set; }
[DataMember(EmitDefaultValue = false)]
Expand Down Expand Up @@ -550,7 +572,26 @@ public class Post : ICloneable
[DataMember(EmitDefaultValue = false)]
public bool should_bypass_tagfiltering { get; set; }
[DataMember(EmitDefaultValue = false)]
public string provider_uri { get; set; }
public bool should_bypass_safemode { get; set; }
[DataMember(EmitDefaultValue = false)]
public bool can_modify_safe_mode { get; set; }
[DataMember(EmitDefaultValue = false)]
public object survey { get; set; }
[DataMember(EmitDefaultValue = false)]
public string url { get; set; }
[DataMember(EmitDefaultValue = false)]
public string link_image { get; set; }
[DataMember(EmitDefaultValue = false)]
public LinkImageDimensions link_image_dimensions { get; set; }
[DataMember(EmitDefaultValue = false)]
public object link_author { get; set; }
[DataMember(EmitDefaultValue = false)]
public string excerpt { get; set; }
[DataMember(EmitDefaultValue = false)]
public string publisher { get; set; }
[DataMember(EmitDefaultValue = false)]
public string description { get; set; }


public object Clone()
{
Expand Down Expand Up @@ -598,7 +639,7 @@ private void Initialize()
thumbnail_width = 0;
thumbnail_height = 0;
duration = 0;
player = new List<Player>();
player = new object();
audio_url = string.Empty;
audio_source_url = string.Empty;
audio_type = string.Empty;
Expand Down Expand Up @@ -659,6 +700,16 @@ private void Initialize()
dialogue = new List<Dialogue>();
is_anonymous = false;
is_submission = false;
should_bypass_safemode = false;
can_modify_safe_mode = false;
survey = new object();
url = string.Empty;
link_image = string.Empty;
link_image_dimensions = new LinkImageDimensions();
link_author = string.Empty;
excerpt = string.Empty;
publisher = string.Empty;
description = string.Empty;
}
}

Expand Down Expand Up @@ -693,6 +744,24 @@ public class Player
public string embed_code { get; set; }
}

[DataContract]
public class Youtube
{
[DataMember(EmitDefaultValue = false)]
public string video_id { get; set; }
[DataMember(EmitDefaultValue = false)]
public int width { get; set; }
[DataMember(EmitDefaultValue = false)]
public int height { get; set; }
}

[DataContract]
public class Video
{
[DataMember(EmitDefaultValue = false)]
public Youtube youtube { get; set; }
}

[DataContract]
public class TrackingHtml
{
Expand Down

0 comments on commit 3ea452d

Please sign in to comment.