Skip to content
Browse files

Removed Subproject

  • Loading branch information...
1 parent 5d26ae8 commit 6591689db89127059f542abc1c3f030e392a6894 @pjmagee committed Oct 4, 2012
Showing with 275 additions and 276 deletions.
  1. +0 −1 4chandownloader
  2. +91 −91 4chandownloader.sln
  3. +20 −20 Downloader.Console/config.xml
  4. +164 −164 Downloader/Board.cs
1 4chandownloader
@@ -1 +0,0 @@
-Subproject commit 70df7b970f18bbdef12d089e3a366b152bb16d32
View
182 4chandownloader.sln
@@ -1,91 +1,91 @@
-
-Microsoft Visual Studio Solution File, Format Version 11.00
-# Visual Studio 2010
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Downloader", "Downloader\Downloader.csproj", "{B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}"
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{E2A1E721-B4CB-44CD-995F-BEEEAA611A58}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Downloader.GUI", "Downloader.GUI\Downloader.GUI.csproj", "{B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}"
- ProjectSection(ProjectDependencies) = postProject
- {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC} = {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}
- EndProjectSection
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Downloader.Tests", "Downloader.Tests\Downloader.Tests.csproj", "{78144C04-B1A9-45CA-916E-47C17CF639D0}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Downloader.Console", "Downloader.Console\Downloader.Console.csproj", "{6088982E-820C-4885-BA8B-9E31DE269568}"
- ProjectSection(ProjectDependencies) = postProject
- {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC} = {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}
- EndProjectSection
-EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{F5A6635D-A6D9-4CE4-B74C-E8E7EC627CEE}"
- ProjectSection(SolutionItems) = preProject
- .nuget\NuGet.exe = .nuget\NuGet.exe
- .nuget\NuGet.targets = .nuget\NuGet.targets
- EndProjectSection
-EndProject
-Global
- GlobalSection(TestCaseManagementSettings) = postSolution
- CategoryFile = 4chandownloader-v3.vsmdi
- EndGlobalSection
- GlobalSection(SolutionConfigurationPlatforms) = preSolution
- Debug|Any CPU = Debug|Any CPU
- Debug|Mixed Platforms = Debug|Mixed Platforms
- Debug|x86 = Debug|x86
- Release|Any CPU = Release|Any CPU
- Release|Mixed Platforms = Release|Mixed Platforms
- Release|x86 = Release|x86
- EndGlobalSection
- GlobalSection(ProjectConfigurationPlatforms) = postSolution
- {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
- {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Debug|Any CPU.Build.0 = Debug|Any CPU
- {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
- {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
- {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Debug|x86.ActiveCfg = Debug|Any CPU
- {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Debug|x86.Build.0 = Debug|Any CPU
- {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Release|Any CPU.ActiveCfg = Release|Any CPU
- {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Release|Any CPU.Build.0 = Release|Any CPU
- {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
- {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Release|Mixed Platforms.Build.0 = Release|Any CPU
- {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Release|x86.ActiveCfg = Release|Any CPU
- {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Release|x86.Build.0 = Release|Any CPU
- {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Debug|Any CPU.ActiveCfg = Debug|x86
- {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Debug|Any CPU.Build.0 = Debug|x86
- {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Debug|Mixed Platforms.ActiveCfg = Debug|x86
- {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Debug|Mixed Platforms.Build.0 = Debug|x86
- {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Debug|x86.ActiveCfg = Debug|x86
- {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Debug|x86.Build.0 = Debug|x86
- {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Release|Any CPU.ActiveCfg = Release|x86
- {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Release|Any CPU.Build.0 = Release|x86
- {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Release|Mixed Platforms.ActiveCfg = Release|x86
- {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Release|Mixed Platforms.Build.0 = Release|x86
- {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Release|x86.ActiveCfg = Release|x86
- {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Release|x86.Build.0 = Release|x86
- {78144C04-B1A9-45CA-916E-47C17CF639D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
- {78144C04-B1A9-45CA-916E-47C17CF639D0}.Debug|Any CPU.Build.0 = Debug|Any CPU
- {78144C04-B1A9-45CA-916E-47C17CF639D0}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
- {78144C04-B1A9-45CA-916E-47C17CF639D0}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
- {78144C04-B1A9-45CA-916E-47C17CF639D0}.Debug|x86.ActiveCfg = Debug|Any CPU
- {78144C04-B1A9-45CA-916E-47C17CF639D0}.Debug|x86.Build.0 = Debug|Any CPU
- {78144C04-B1A9-45CA-916E-47C17CF639D0}.Release|Any CPU.ActiveCfg = Release|Any CPU
- {78144C04-B1A9-45CA-916E-47C17CF639D0}.Release|Any CPU.Build.0 = Release|Any CPU
- {78144C04-B1A9-45CA-916E-47C17CF639D0}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
- {78144C04-B1A9-45CA-916E-47C17CF639D0}.Release|Mixed Platforms.Build.0 = Release|Any CPU
- {78144C04-B1A9-45CA-916E-47C17CF639D0}.Release|x86.ActiveCfg = Release|Any CPU
- {78144C04-B1A9-45CA-916E-47C17CF639D0}.Release|x86.Build.0 = Release|Any CPU
- {6088982E-820C-4885-BA8B-9E31DE269568}.Debug|Any CPU.ActiveCfg = Debug|x86
- {6088982E-820C-4885-BA8B-9E31DE269568}.Debug|Any CPU.Build.0 = Debug|x86
- {6088982E-820C-4885-BA8B-9E31DE269568}.Debug|Mixed Platforms.ActiveCfg = Debug|x86
- {6088982E-820C-4885-BA8B-9E31DE269568}.Debug|Mixed Platforms.Build.0 = Debug|x86
- {6088982E-820C-4885-BA8B-9E31DE269568}.Debug|x86.ActiveCfg = Debug|x86
- {6088982E-820C-4885-BA8B-9E31DE269568}.Debug|x86.Build.0 = Debug|x86
- {6088982E-820C-4885-BA8B-9E31DE269568}.Release|Any CPU.ActiveCfg = Release|x86
- {6088982E-820C-4885-BA8B-9E31DE269568}.Release|Any CPU.Build.0 = Release|x86
- {6088982E-820C-4885-BA8B-9E31DE269568}.Release|Mixed Platforms.ActiveCfg = Release|x86
- {6088982E-820C-4885-BA8B-9E31DE269568}.Release|Mixed Platforms.Build.0 = Release|x86
- {6088982E-820C-4885-BA8B-9E31DE269568}.Release|x86.ActiveCfg = Release|x86
- {6088982E-820C-4885-BA8B-9E31DE269568}.Release|x86.Build.0 = Release|x86
- EndGlobalSection
- GlobalSection(SolutionProperties) = preSolution
- HideSolutionNode = FALSE
- EndGlobalSection
-EndGlobal
+
+Microsoft Visual Studio Solution File, Format Version 11.00
+# Visual Studio 2012
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Downloader", "Downloader\Downloader.csproj", "{B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{E2A1E721-B4CB-44CD-995F-BEEEAA611A58}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Downloader.GUI", "Downloader.GUI\Downloader.GUI.csproj", "{B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}"
+ ProjectSection(ProjectDependencies) = postProject
+ {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC} = {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}
+ EndProjectSection
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Downloader.Tests", "Downloader.Tests\Downloader.Tests.csproj", "{78144C04-B1A9-45CA-916E-47C17CF639D0}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Downloader.Console", "Downloader.Console\Downloader.Console.csproj", "{6088982E-820C-4885-BA8B-9E31DE269568}"
+ ProjectSection(ProjectDependencies) = postProject
+ {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC} = {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}
+ EndProjectSection
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{F5A6635D-A6D9-4CE4-B74C-E8E7EC627CEE}"
+ ProjectSection(SolutionItems) = preProject
+ .nuget\NuGet.exe = .nuget\NuGet.exe
+ .nuget\NuGet.targets = .nuget\NuGet.targets
+ EndProjectSection
+EndProject
+Global
+ GlobalSection(TestCaseManagementSettings) = postSolution
+ CategoryFile = 4chandownloader-v3.vsmdi
+ EndGlobalSection
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Any CPU = Debug|Any CPU
+ Debug|Mixed Platforms = Debug|Mixed Platforms
+ Debug|x86 = Debug|x86
+ Release|Any CPU = Release|Any CPU
+ Release|Mixed Platforms = Release|Mixed Platforms
+ Release|x86 = Release|x86
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+ {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+ {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Debug|x86.Build.0 = Debug|Any CPU
+ {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Release|Any CPU.Build.0 = Release|Any CPU
+ {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+ {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+ {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Release|x86.ActiveCfg = Release|Any CPU
+ {B3A3CBF0-1D72-4753-A56F-5CC6803B86AC}.Release|x86.Build.0 = Release|Any CPU
+ {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Debug|Any CPU.ActiveCfg = Debug|x86
+ {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Debug|Any CPU.Build.0 = Debug|x86
+ {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Debug|Mixed Platforms.ActiveCfg = Debug|x86
+ {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Debug|Mixed Platforms.Build.0 = Debug|x86
+ {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Debug|x86.ActiveCfg = Debug|x86
+ {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Debug|x86.Build.0 = Debug|x86
+ {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Release|Any CPU.ActiveCfg = Release|x86
+ {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Release|Any CPU.Build.0 = Release|x86
+ {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Release|Mixed Platforms.ActiveCfg = Release|x86
+ {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Release|Mixed Platforms.Build.0 = Release|x86
+ {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Release|x86.ActiveCfg = Release|x86
+ {B0100FD3-5E1B-4224-AE12-1E25D4D0F83F}.Release|x86.Build.0 = Release|x86
+ {78144C04-B1A9-45CA-916E-47C17CF639D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {78144C04-B1A9-45CA-916E-47C17CF639D0}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {78144C04-B1A9-45CA-916E-47C17CF639D0}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+ {78144C04-B1A9-45CA-916E-47C17CF639D0}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+ {78144C04-B1A9-45CA-916E-47C17CF639D0}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {78144C04-B1A9-45CA-916E-47C17CF639D0}.Debug|x86.Build.0 = Debug|Any CPU
+ {78144C04-B1A9-45CA-916E-47C17CF639D0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {78144C04-B1A9-45CA-916E-47C17CF639D0}.Release|Any CPU.Build.0 = Release|Any CPU
+ {78144C04-B1A9-45CA-916E-47C17CF639D0}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+ {78144C04-B1A9-45CA-916E-47C17CF639D0}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+ {78144C04-B1A9-45CA-916E-47C17CF639D0}.Release|x86.ActiveCfg = Release|Any CPU
+ {78144C04-B1A9-45CA-916E-47C17CF639D0}.Release|x86.Build.0 = Release|Any CPU
+ {6088982E-820C-4885-BA8B-9E31DE269568}.Debug|Any CPU.ActiveCfg = Debug|x86
+ {6088982E-820C-4885-BA8B-9E31DE269568}.Debug|Any CPU.Build.0 = Debug|x86
+ {6088982E-820C-4885-BA8B-9E31DE269568}.Debug|Mixed Platforms.ActiveCfg = Debug|x86
+ {6088982E-820C-4885-BA8B-9E31DE269568}.Debug|Mixed Platforms.Build.0 = Debug|x86
+ {6088982E-820C-4885-BA8B-9E31DE269568}.Debug|x86.ActiveCfg = Debug|x86
+ {6088982E-820C-4885-BA8B-9E31DE269568}.Debug|x86.Build.0 = Debug|x86
+ {6088982E-820C-4885-BA8B-9E31DE269568}.Release|Any CPU.ActiveCfg = Release|x86
+ {6088982E-820C-4885-BA8B-9E31DE269568}.Release|Any CPU.Build.0 = Release|x86
+ {6088982E-820C-4885-BA8B-9E31DE269568}.Release|Mixed Platforms.ActiveCfg = Release|x86
+ {6088982E-820C-4885-BA8B-9E31DE269568}.Release|Mixed Platforms.Build.0 = Release|x86
+ {6088982E-820C-4885-BA8B-9E31DE269568}.Release|x86.ActiveCfg = Release|x86
+ {6088982E-820C-4885-BA8B-9E31DE269568}.Release|x86.Build.0 = Release|x86
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+EndGlobal
View
40 Downloader.Console/config.xml
@@ -1,21 +1,21 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Config xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
- <boards>
- <board name="a" pages="5" folders="true" continuous="true" directory="C:\Temp\" disabled="true" snooze="30">
- <timer days="0" hours="0" minutes="0" seconds="0" disabled="true" />
- <constraint minimum="0" disabled="true" />
- </board>
- <board name="b" pages="5" folders="true" continuous="true" directory="C:\Temp\" disabled="true" snooze="30">
- <timer days="0" hours="0" minutes="0" seconds="0" disabled="true" />
- <constraint minimum="0" disabled="true" />
- </board>
- <board name="c" pages="5" folders="true" continuous="true" directory="C:\Temp\" disabled="true" snooze="30">
- <timer days="0" hours="0" minutes="0" seconds="0" disabled="true" />
- <constraint minimum="0" disabled="false" />
- </board>
- <board name="d" pages="5" folders="true" continuous="true" directory="C:\Temp\" disabled="true" snooze="30">
- <timer days="0" hours="0" minutes="20" seconds="0" disabled="true" />
- <constraint minimum="0" disabled="true" />
- </board>
- </boards>
+<?xml version="1.0" encoding="utf-8"?>
+<Config xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <boards>
+ <board name="a" pages="5" folders="true" continuous="true" directory="C:\Temp\" disabled="true" snooze="30">
+ <timer days="0" hours="0" minutes="0" seconds="0" disabled="true" />
+ <constraint minimum="0" disabled="true" />
+ </board>
+ <board name="b" pages="5" folders="true" continuous="true" directory="C:\Temp\" disabled="false" snooze="30">
+ <timer days="0" hours="0" minutes="0" seconds="0" disabled="true" />
+ <constraint minimum="0" disabled="true" />
+ </board>
+ <board name="c" pages="5" folders="true" continuous="true" directory="C:\Temp\" disabled="true" snooze="30">
+ <timer days="0" hours="0" minutes="0" seconds="0" disabled="true" />
+ <constraint minimum="0" disabled="false" />
+ </board>
+ <board name="d" pages="5" folders="true" continuous="true" directory="C:\Temp\" disabled="true" snooze="30">
+ <timer days="0" hours="0" minutes="20" seconds="0" disabled="true" />
+ <constraint minimum="0" disabled="true" />
+ </board>
+ </boards>
</Config>
View
328 Downloader/Board.cs
@@ -1,164 +1,164 @@
-using System;
-using System.Collections.Generic;
-using System.Collections.Concurrent;
-using System.IO;
-using System.Linq;
-using System.Net;
-using System.Text.RegularExpressions;
-using System.Threading.Tasks;
-
-namespace Downloader
-{
- public class Board
- {
- public string Name { get; set; }
- public bool Repeat { get; set; }
- public int MaxPage { get; set; }
- public bool CreateFolders { get; set; }
- public long? Ticks { get; set; }
- public List<BoardThread> BoardThreads { get; set; }
- public ConcurrentBag<string> Threads { get; set; }
- public int? MinImages { get; set; }
- public int? Sleep { get; set; }
-
- private DirectoryInfo _directoryInfo;
- public DirectoryInfo DirectoryInfo
- {
- get
- {
- if(!_directoryInfo.Exists)
- {
- _directoryInfo.Create();
- }
- return _directoryInfo;
- }
- set { _directoryInfo = value; }
- }
-
- private const string ThreadImageRegex = "\\d{13}.(jpg|png|gif)"; // contains 13 digits followed by a . and ends in a jpg or png or gif
- private const string ThreadLinkRegex = "<a href=\"res/\\d{1,12}\">Reply</a>";
- private const string ThreadNameRegex = "\\d{1,12}"; // contains digits 1 to 12 long
- private string ImageLinkRegex
- {
- get { return "<a href=\"//images.4chan.org/" + Name + "/src/\\d{10,16}.(jpg|png|gif)\" target=\"_blank\">"; }
- }
-
- public Board(string name = "", bool repeat = false, int maxPage = 5, bool createFolders = true)
- {
- Name = name;
- Repeat = repeat;
- MaxPage = maxPage;
- CreateFolders = createFolders;
- BoardThreads = new List<BoardThread>();
- Threads = new ConcurrentBag<string>();
- }
-
- public void GatherThreads()
- {
- // Page 0 -> Page 15
- Parallel.For(fromInclusive: 0, toExclusive: MaxPage, body: (i, a) =>
- {
- try
- {
- Console.WriteLine(string.Format("STGR http://boards.4chan.org/{0}/{1}", Name, i));
- string html = new WebClient().DownloadString(string.Format("http://boards.4chan.org/{0}/{1}", Name, i));
- var matchCollection = Regex.Matches(html, ThreadLinkRegex);
- matchCollection.Cast<Match>().ToList().ForEach(thread => Threads.Add(Regex.Match(thread.Value, ThreadNameRegex).Value));
- Console.WriteLine(string.Format("FNGT http://boards.4chan.org/{0}/{1}", Name, i));
- }
- catch(Exception e)
- {
- Console.WriteLine(string.Format("FTGT http://boards.4chan.org/{0}/{1}][{2}", Name, i, e.Message));
- }
- });
- }
-
- public void ProcessThreads()
- {
- var threadFolders = Threads
- .Select(thread => new BoardThread()
- {
- Parent = this,
- Name = thread,
- DirectoryInfo = CreateFolders ? DirectoryInfo.CreateSubdirectory(thread) : DirectoryInfo,
- });
-
- BoardThreads = new List<BoardThread>(threadFolders);
-
- var groups = BoardThreads.Select((x, i) => new { Index = i, Value = x })
- .GroupBy(x => x.Index / 5) // Sets of 5 would be nice...
- .Select(x => x.Select(v => v.Value).ToList()).ToList();
-
- foreach(var group in groups)
- {
- Parallel.ForEach(source: group, body: ProcessThread);
- }
-
-
- }
-
-
- private void ProcessThread(BoardThread thread)
- {
- try
- {
- string html = new WebClient().DownloadString(address: new Uri(string.Format(@"http://boards.4chan.org/{0}/res/{1}", Name, thread.Name)));
-
- MatchCollection links = Regex.Matches(html, ImageLinkRegex, RegexOptions.IgnoreCase);
-
- if(MinImages.HasValue)
- {
- if(links.Count < MinImages.Value)
- {
- Console.WriteLine(string.Format(@"DNMF http://boards.4chan.org/{0}/res/{1}", Name, thread.Name));
-
- if(!thread.DirectoryInfo.Equals(DirectoryInfo))
- thread.DirectoryInfo.Delete();
-
- return;
- }
- }
-
- var groups = links.Cast<Match>().AsParallel().Select((x, i) => new { Index = i, Value = x })
- .GroupBy(x => x.Index / 5) // Sets of 10 would be nice...
- .Select(x => x.Select(v => v.Value).ToList()).ToList();
-
- foreach (var group in groups)
- {
- Console.WriteLine("Processing the next 5 images...");
- // Send in a set of 10 at once...
- Parallel.ForEach(group, link =>
- {
- string image = Regex.Match(link.Value, ThreadImageRegex).Value;
-
- try
- {
- Console.WriteLine(string.Format(@"DLNG http://images.4chan.org/{0}/src/{1}", Name, image));
- new WebClient().DownloadFile(new Uri(string.Format(@"http://images.4chan.org/{0}/src/{1}", Name, image)), thread.DirectoryInfo.FullName + "\\" + image);
- }
- catch (WebException e)
- {
- var httpWebResponse = e.Response as HttpWebResponse;
-
- if (httpWebResponse != null)
- {
- Console.WriteLine(string.Format(@"FTDL http://images.4chan.org/{0}/src/{1} - {2}", Name, image, (int)(httpWebResponse).StatusCode));
- }
- }
- });
- }
- }
- catch (WebException e)
- {
- var httpWebResponse = e.Response as HttpWebResponse;
-
- if (httpWebResponse != null)
- {
- Console.WriteLine(string.Format(@"FTDL http://boards.4chan.org/{0}/res/{1} - {2}", Name, thread.Name, (int)(httpWebResponse).StatusCode));
- }
- }
-
- }
-
- }
-}
+using System;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+
+namespace Downloader
+{
+ public class Board
+ {
+ public string Name { get; set; }
+ public bool Repeat { get; set; }
+ public int MaxPage { get; set; }
+ public bool CreateFolders { get; set; }
+ public long? Ticks { get; set; }
+ public List<BoardThread> BoardThreads { get; set; }
+ public ConcurrentBag<string> Threads { get; set; }
+ public int? MinImages { get; set; }
+ public int? Sleep { get; set; }
+
+ private DirectoryInfo _directoryInfo;
+ public DirectoryInfo DirectoryInfo
+ {
+ get
+ {
+ if(!_directoryInfo.Exists)
+ {
+ _directoryInfo.Create();
+ }
+ return _directoryInfo;
+ }
+ set { _directoryInfo = value; }
+ }
+
+ private const string ThreadImageRegex = "\\d{13}.(jpg|png|gif)"; // contains 13 digits followed by a . and ends in a jpg or png or gif
+ private const string ThreadLinkRegex = "<a href=\"res/\\d{1,12}\" class=\"replylink\" >Reply</a>";
+ private const string ThreadNameRegex = "\\d{1,12}"; // contains digits 1 to 12 long
+ private string ImageLinkRegex
+ {
+ get { return "<a href=\"//images.4chan.org/" + Name + "/src/\\d{10,16}.(jpg|png|gif)\" target=\"_blank\">"; }
+ }
+
+ public Board(string name = "", bool repeat = false, int maxPage = 5, bool createFolders = true)
+ {
+ Name = name;
+ Repeat = repeat;
+ MaxPage = maxPage;
+ CreateFolders = createFolders;
+ BoardThreads = new List<BoardThread>();
+ Threads = new ConcurrentBag<string>();
+ }
+
+ public void GatherThreads()
+ {
+ // Page 0 -> Page 15
+ Parallel.For(fromInclusive: 0, toExclusive: MaxPage, body: (i, a) =>
+ {
+ try
+ {
+ Console.WriteLine(string.Format("STGR http://boards.4chan.org/{0}/{1}", Name, i));
+ string html = new WebClient().DownloadString(string.Format("http://boards.4chan.org/{0}/{1}", Name, i));
+ var matchCollection = Regex.Matches(html, ThreadLinkRegex);
+ matchCollection.Cast<Match>().ToList().ForEach(thread => Threads.Add(Regex.Match(thread.Value, ThreadNameRegex).Value));
+ Console.WriteLine(string.Format("FNGT http://boards.4chan.org/{0}/{1}", Name, i));
+ }
+ catch(Exception e)
+ {
+ Console.WriteLine(string.Format("FTGT http://boards.4chan.org/{0}/{1}][{2}", Name, i, e.Message));
+ }
+ });
+ }
+
+ public void ProcessThreads()
+ {
+ var threadFolders = Threads
+ .Select(thread => new BoardThread()
+ {
+ Parent = this,
+ Name = thread,
+ DirectoryInfo = CreateFolders ? DirectoryInfo.CreateSubdirectory(thread) : DirectoryInfo,
+ });
+
+ BoardThreads = new List<BoardThread>(threadFolders);
+
+ var groups = BoardThreads.Select((x, i) => new { Index = i, Value = x })
+ .GroupBy(x => x.Index / 5) // Sets of 5 would be nice...
+ .Select(x => x.Select(v => v.Value).ToList()).ToList();
+
+ foreach(var group in groups)
+ {
+ Parallel.ForEach(source: group, body: ProcessThread);
+ }
+
+
+ }
+
+
+ private void ProcessThread(BoardThread thread)
+ {
+ try
+ {
+ string html = new WebClient().DownloadString(address: new Uri(string.Format(@"http://boards.4chan.org/{0}/res/{1}", Name, thread.Name)));
+
+ MatchCollection links = Regex.Matches(html, ImageLinkRegex, RegexOptions.IgnoreCase);
+
+ if(MinImages.HasValue)
+ {
+ if(links.Count < MinImages.Value)
+ {
+ Console.WriteLine(string.Format(@"DNMF http://boards.4chan.org/{0}/res/{1}", Name, thread.Name));
+
+ if(!thread.DirectoryInfo.Equals(DirectoryInfo))
+ thread.DirectoryInfo.Delete();
+
+ return;
+ }
+ }
+
+ var groups = links.Cast<Match>().AsParallel().Select((x, i) => new { Index = i, Value = x })
+ .GroupBy(x => x.Index / 5) // Sets of 10 would be nice...
+ .Select(x => x.Select(v => v.Value).ToList()).ToList();
+
+ foreach (var group in groups)
+ {
+ Console.WriteLine("Processing the next 5 images...");
+ // Send in a set of 10 at once...
+ Parallel.ForEach(group, link =>
+ {
+ string image = Regex.Match(link.Value, ThreadImageRegex).Value;
+
+ try
+ {
+ Console.WriteLine(string.Format(@"DLNG http://images.4chan.org/{0}/src/{1}", Name, image));
+ new WebClient().DownloadFile(new Uri(string.Format(@"http://images.4chan.org/{0}/src/{1}", Name, image)), thread.DirectoryInfo.FullName + "\\" + image);
+ }
+ catch (WebException e)
+ {
+ var httpWebResponse = e.Response as HttpWebResponse;
+
+ if (httpWebResponse != null)
+ {
+ Console.WriteLine(string.Format(@"FTDL http://images.4chan.org/{0}/src/{1} - {2}", Name, image, (int)(httpWebResponse).StatusCode));
+ }
+ }
+ });
+ }
+ }
+ catch (WebException e)
+ {
+ var httpWebResponse = e.Response as HttpWebResponse;
+
+ if (httpWebResponse != null)
+ {
+ Console.WriteLine(string.Format(@"FTDL http://boards.4chan.org/{0}/res/{1} - {2}", Name, thread.Name, (int)(httpWebResponse).StatusCode));
+ }
+ }
+
+ }
+
+ }
+}

0 comments on commit 6591689

Please sign in to comment.
Something went wrong with that request. Please try again.