-
Notifications
You must be signed in to change notification settings - Fork 0
/
[PA 4] DashboardUpdate.cs
114 lines (102 loc) · 4.25 KB
/
[PA 4] DashboardUpdate.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Queue;
using Microsoft.WindowsAzure.Storage.Table;
using System;
using System.Collections.Generic;
using System.Configuration;
namespace Share
{
public class DashboardUpdate
{
public static DashboardUpdate instance;
private CloudTable dashboardTable;
private CloudTable pageTable;
private CloudTable errorTable;
private CloudTable mappingTable;
private CloudQueue urlQ;
public static DashboardUpdate GetInstance()
{
if (instance == null)
{
instance = new DashboardUpdate(
Azure.GetInstance().getTableReference(ConfigurationManager.AppSettings["dashboardInfo"]),
Azure.GetInstance().getTableReference(ConfigurationManager.AppSettings["pageInfo"]),
Azure.GetInstance().getTableReference(ConfigurationManager.AppSettings["errorInfo"]),
Azure.GetInstance().getTableReference(ConfigurationManager.AppSettings["mappingInfo"]),
Azure.GetInstance().getQueueReference(ConfigurationManager.AppSettings["urlQ"])
);
}
return instance;
}
public DashboardUpdate(CloudTable generalInfoTable, CloudTable pageTable, CloudTable errorTable, CloudTable mappingTable, CloudQueue urlQ)
{
this.dashboardTable = generalInfoTable;
this.pageTable = pageTable;
this.errorTable = errorTable;
this.mappingTable = mappingTable;
this.urlQ = urlQ;
IEnumerator<Dashboard> results = generalInfoTable.ExecuteQuery(new TableQuery<Dashboard>()).GetEnumerator();
if (!results.MoveNext())
{
generalInfoTable.Execute(TableOperation.InsertOrReplace(new Dashboard(0, 0, 0, new List<string>(), new List<string>())));
}
}
public void clear()
{
Dashboard dInfo = new Dashboard(0, 0, 0, new List<string>(), new List<string>());
dInfo.ETag = "*";
dashboardTable.Execute(TableOperation.Replace(dInfo));
}
public Dashboard getProfile()
{
Dashboard dProfile = new Dashboard(0, 0, 0, new List<string>(), new List<string>());
try
{
IEnumerator<Dashboard> results = dashboardTable.ExecuteQuery(new TableQuery<Dashboard>()).GetEnumerator();
if (results.MoveNext())
{
Dashboard result = results.Current;
dProfile.numUrlsCrawled = result.numUrlsCrawled;
dProfile.numUrlsIndexed = result.numUrlsIndexed;
}
foreach (PageInfo webinfo in pageTable.ExecuteQuery((new TableQuery<PageInfo>()).Take(10)))
{
dProfile.urlsCrawled.Add(webinfo.url);
}
foreach (ErrorUrl crawlError in errorTable.ExecuteQuery((new TableQuery<ErrorUrl>()).Take(100)))
{
dProfile.crawlErrors.Add(crawlError.message + ": " + crawlError.url);
}
urlQ.FetchAttributes();
dProfile.numUrlsQueued = urlQ.ApproximateMessageCount.GetValueOrDefault(0);
}
catch (StorageException) { }
return dProfile;
}
public void increaseStats(int urlsCrawled, int urlsIndexed)
{
int retryCount = 10;
while (retryCount > 0)
{
try
{
IEnumerator<Dashboard> results = dashboardTable.ExecuteQuery(new TableQuery<Dashboard>()).GetEnumerator();
if (!results.MoveNext())
{
return;
}
Dashboard dInfo = results.Current;
dInfo.numUrlsCrawled += urlsCrawled;
dInfo.numUrlsIndexed += urlsIndexed;
dashboardTable.Execute(TableOperation.Replace(dInfo));
}
catch (StorageException)
{
retryCount--;
continue;
}
break;
}
}
}
}