-
Notifications
You must be signed in to change notification settings - Fork 556
/
RobotsDotTextFinder.cs
53 lines (46 loc) · 1.85 KB
/
RobotsDotTextFinder.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
using System;
using System.Net;
using System.Threading.Tasks;
using Serilog;
namespace Abot2.Core
{
/// <summary>
/// Finds and builds the robots.txt file abstraction
/// </summary>
public interface IRobotsDotTextFinder
{
/// <summary>
/// Finds the robots.txt file using the rootUri.
/// If rootUri is http://yahoo.com, it will look for robots at http://yahoo.com/robots.txt.
/// If rootUri is http://music.yahoo.com, it will look for robots at http://music.yahoo.com/robots.txt
/// </summary>
/// <param name="rootUri">The root domain</param>
/// <returns>Object representing the robots.txt file or returns null</returns>
Task<IRobotsDotText> FindAsync(Uri rootUri);
}
public class RobotsDotTextFinder : IRobotsDotTextFinder
{
IPageRequester _pageRequester;
public RobotsDotTextFinder(IPageRequester pageRequester)
{
_pageRequester = pageRequester ?? throw new ArgumentNullException(nameof(pageRequester));
}
public async Task<IRobotsDotText> FindAsync(Uri rootUri)
{
if (rootUri == null)
throw new ArgumentNullException(nameof(rootUri));
var robotsUri = new Uri(rootUri, "/robots.txt");
var page = await _pageRequester.MakeRequestAsync(robotsUri).ConfigureAwait(false);
if (page == null ||
page.HttpRequestException != null ||
page.HttpResponseMessage == null ||
page.HttpResponseMessage.StatusCode != HttpStatusCode.OK)
{
Log.Debug("Did not find robots.txt file at [{0}]", robotsUri);
return null;
}
Log.Debug("Found robots.txt file at [{0}]", robotsUri);
return new RobotsDotText(rootUri, page.Content.Text);
}
}
}