Skip to content

Commit

Permalink
NAS-2638: Merge and fix of NAS-2642. Edited some buttons and removed …
Browse files Browse the repository at this point in the history 
…tomcat articats from WEB-INF/lib directory.
  • Loading branch information
@nclarkekb
nclarkekb committed Oct 25, 2017
1 parent 839e02b commit 0b21700
Show file tree
Hide file tree
Showing 7 changed files with 186 additions and 80 deletions.
3 changes: 3 additions & 0 deletions harvester/heritrix3/heritrix3-monitor/src/main/assembly/assembly.xml
View file
Expand Up @@ -56,6 +56,9 @@
<exclude>org.hibernate:*:jar</exclude>
<exclude>antlr:antlr:jar</exclude>
<exclude>commons-collections:commons-collections:jar</exclude>
<!-- tomcat -->
<exclude>org.apache.tomcat.embed:*:jar</exclude>
<exclude>org.apache.tomcat:*:jar</exclude>
<!-- jetty -->
<exclude>org.eclipse.jetty:*:jar</exclude>
<exclude>org.eclipse.jetty.websocket:*:jar</exclude>
Expand Down
5 changes: 3 additions & 2 deletions ...rix3-monitor/src/main/java/dk/netarkivet/heritrix3/monitor/Heritrix3JobMonitorThread.java
View file
Expand Up @@ -22,7 +22,6 @@
import dk.netarkivet.harvester.datamodel.HarvestChannelDAO;
import dk.netarkivet.harvester.datamodel.JobDAO;
import dk.netarkivet.harvester.datamodel.RunningJobsInfoDAO;
import dk.netarkivet.harvester.harvesting.monitor.HarvestMonitor;

public class Heritrix3JobMonitorThread implements Runnable {

Expand All @@ -33,7 +32,7 @@ public class Heritrix3JobMonitorThread implements Runnable {
private NASEnvironment environment;

/** <code>HarvestMonitor</code> instance. */
public static HarvestMonitor harvestMonitor;
//public static HarvestMonitor harvestMonitor;

/** <code>JobDAO</code> instance. */
public static JobDAO jobDAO;
Expand Down Expand Up @@ -71,9 +70,11 @@ public Heritrix3JobMonitorThread(NASEnvironment environment) {
}

public synchronized void init() throws Exception {
/*
if (harvestMonitor == null) {
harvestMonitor = HarvestMonitor.getInstance();
}
*/
if (jobDAO == null) {
jobDAO = JobDAO.getInstance();
}
Expand Down
11 changes: 4 additions & 7 deletions ...tor/src/main/java/dk/netarkivet/heritrix3/monitor/resources/H3CrawlLogCachedResource.java
View file
Expand Up @@ -124,10 +124,11 @@ public void crawllog_list(HttpServletRequest req, HttpServletResponse resp, List
SearchResult searchResult = null;

if (q != null) {

searchResult = h3Job.getSearchResult(q);
searchResult.update();
pageable = searchResult;
} else {
q = ".*";
}

lines = pageable.getIndexSize();
Expand Down Expand Up @@ -160,18 +161,14 @@ public void crawllog_list(HttpServletRequest req, HttpServletResponse resp, List
sb.append("</a>");
//sb.append("the cache manually ");
sb.append("</div>\n");

sb.append("<div style=\"clear:both;\"></div>\n");
sb.append("</div>\n");

if (q == null) {
q = ".*";
}

sb.append("<div style=\"margin-bottom:20px;\">\n");

sb.append("<form class=\"form-horizontal\" action=\"?\" name=\"insert_form\" method=\"post\" enctype=\"application/x-www-form-urlencoded\" accept-charset=\"utf-8\">");
sb.append("<label for=\"itemsperpage\">Lines to show:</label>");
sb.append("<label for=\"itemsperpage\">Lines per page:</label>");
sb.append("<input type=\"text\" id=\"itemsperpage\" name=\"itemsperpage\" value=\"" + linesPerPage + "\" placeholder=\"must be &gt; 25 and &lt; 1000 \">\n");
sb.append("<label for=\"q\">Filter regex:</label>");
sb.append("<input type=\"text\" id=\"q\" name=\"q\" value=\"" + q + "\" placeholder=\"content-type\" style=\"display:inline;width:350px;\">\n");
Expand Down
124 changes: 103 additions & 21 deletions ...3-monitor/src/main/java/dk/netarkivet/heritrix3/monitor/resources/H3FrontierResource.java
View file
@@ -1,8 +1,11 @@
package dk.netarkivet.heritrix3.monitor.resources;

import java.io.IOException;
import java.net.URLEncoder;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.servlet.ServletContext;
import javax.servlet.ServletOutputStream;
Expand All @@ -19,6 +22,7 @@
import dk.netarkivet.heritrix3.monitor.Heritrix3JobMonitor;
import dk.netarkivet.heritrix3.monitor.NASEnvironment;
import dk.netarkivet.heritrix3.monitor.NASUser;
import dk.netarkivet.heritrix3.monitor.Pagination;
import dk.netarkivet.heritrix3.monitor.ResourceAbstract;
import dk.netarkivet.heritrix3.monitor.ResourceManagerAbstract;

Expand Down Expand Up @@ -65,42 +69,67 @@ public void frontier_list(HttpServletRequest req, HttpServletResponse resp, List

StringBuilder sb = new StringBuilder();

String regex = req.getParameter("regex");
if (regex == null || regex.length() == 0) {
regex =".*";
long lines;
long linesPerPage = 100;
long page = 1;
long pages = 0;
String q = null;

String tmpStr;
tmpStr = req.getParameter("page");
if (tmpStr != null && tmpStr.length() > 0) {
try {
page = Long.parseLong(tmpStr);
} catch (NumberFormatException e) {
}
}
long limit = 1000;
String limitStr = req.getParameter("limit");
if (limitStr != null && limitStr.length() > 0) {
tmpStr = req.getParameter("itemsperpage");
if (tmpStr != null && tmpStr.length() > 0) {
try {
limit = Long.parseLong(limitStr);
linesPerPage = Long.parseLong(tmpStr);
} catch (NumberFormatException e) {
}
}

if (linesPerPage < 25) {
linesPerPage = 25;
}
if (linesPerPage > 1000) {
linesPerPage = 1000;
}

String initials = req.getParameter("initials");
if (initials == null) {
initials = "";
}

tmpStr = req.getParameter("q");
if (tmpStr != null && tmpStr.length() > 0 && !tmpStr.equalsIgnoreCase(".*")) {
q = tmpStr;
} else {
q = "";
}

String additionalParams = "";
if (q.length() > 0) {
additionalParams += "&q=" + URLEncoder.encode(q, "UTF-8");
}
if (initials.length() > 0) {
additionalParams += "&initials=" + URLEncoder.encode(initials, "UTF-8");
}

String script = environment.NAS_GROOVY_SCRIPT;

String deleteStr = req.getParameter("delete");
if (deleteStr != null && "1".equals(deleteStr) && initials != null && initials.length() > 0) {
script += "\n";
script += "\ninitials = \"" + initials + "\"";
script += "\ndeleteFromFrontier '" + regex + "'\n";
script += "\ndeleteFromFrontier '" + q + "'\n";
} else {
script += "\n";
script += "\nlistFrontier '" + regex + "', " + limit + "\n";
script += "\nlistFrontier '" + q + "', " + linesPerPage + ", " + (page - 1) + "\n";
}

// To use, just remove the initial "//" from any one of these lines.
//
//killToeThread 1 //Kill a toe thread by number
//listFrontier '.*stats.*' //List uris in the frontier matching a given regexp
//deleteFromFrontier '.*foobar.*' //Remove uris matching a given regexp from the frontier
//printCrawlLog '.*' //View already crawled lines uris matching a given regexp

long jobId = numerics.get(0);
Heritrix3JobMonitor h3Job = environment.h3JobMonitorThread.getRunningH3Job(jobId);

Expand All @@ -111,18 +140,40 @@ public void frontier_list(HttpServletRequest req, HttpServletResponse resp, List
}

sb.append("<form class=\"form-horizontal\" action=\"?\" name=\"insert_form\" method=\"post\" enctype=\"application/x-www-form-urlencoded\" accept-charset=\"utf-8\">\n");
sb.append("<label for=\"limit\">Lines to show:</label>");
sb.append("<input type=\"text\" id=\"limit\" name=\"limit\" value=\"" + limit + "\" placeholder=\"return limit\">\n");
sb.append("<label for=\"regex\">Filter regex:</label>");
sb.append("<input type=\"text\" id=\"regex\" name=\"regex\" value=\"" + regex + "\" placeholder=\"regex\" style=\"display:inline;width:350px;\">\n");
sb.append("<label for=\"limit\">Lines per page:</label>");
sb.append("<input type=\"text\" id=\"itemsperpage\" name=\"itemsperpage\" value=\"" + linesPerPage + "\" placeholder=\"return limit\">\n");
sb.append("<label for=\"q\">Filter regex:</label>");
sb.append("<input type=\"text\" id=\"q\" name=\"q\" value=\"" + q + "\" placeholder=\"regex\" style=\"display:inline;width:350px;\">\n");
sb.append("<button type=\"submit\" name=\"show\" value=\"1\" class=\"btn btn-success\"><i class=\"icon-white icon-thumbs-up\"></i> Show</button>\n");
sb.append("&nbsp;");
sb.append("<label for=\"initials\">User initials:</label>");
sb.append("<input type=\"text\" id=\"initials\" name=\"initials\" value=\"" + initials + "\" placeholder=\"initials\">\n");
sb.append("<button type=\"submit\" name=\"delete\" value=\"1\" class=\"btn btn-success\"><i class=\"icon-white icon-thumbs-up\"></i> Delete</button>\n");
sb.append("<button type=\"submit\" name=\"delete\" value=\"1\" class=\"btn btn-danger\"><i class=\"icon-white icon-trash\"></i> Delete</button>\n");
sb.append("</form>\n");

ScriptResult scriptResult = h3Job.h3wrapper.ExecuteShellScriptInJob(h3Job.jobResult.job.shortName, "groovy", script);
lines = extractLinesAmount(scriptResult);

if (lines > 0) {
pages = (lines + linesPerPage - 1) / linesPerPage;
if (pages == 0) {
pages = 1;
}
}

if (page > pages) {
page = pages;
}

sb.append("<div style=\"float:left;margin: 20px 0px;\">\n");
sb.append("<span>Matching lines: ");
sb.append(lines);
sb.append(" URIs</span>\n");
sb.append("</div>\n");
sb.append(Pagination.getPagination(page, linesPerPage, pages, false, additionalParams));
sb.append("<div style=\"clear:both;\"></div>");
sb.append("<div>\n");
sb.append("<pre>\n");
//System.out.println(new String(scriptResult.response, "UTF-8"));
if (scriptResult != null && scriptResult.script != null) {
if (scriptResult.script.htmlOutput != null) {
Expand All @@ -138,6 +189,10 @@ public void frontier_list(HttpServletRequest req, HttpServletResponse resp, List
sb.append("</fieldset><br />\n");
}
}
sb.append("</pre>\n");
sb.append("</div>\n");
sb.append(Pagination.getPagination(page, linesPerPage, pages, false, additionalParams));
sb.append("</form>");
} else {
sb.append("Job ");
sb.append(jobId);
Expand All @@ -154,4 +209,31 @@ public void frontier_list(HttpServletRequest req, HttpServletResponse resp, List
out.close();
}

private Long extractLinesAmount(ScriptResult scriptResult) {
Pattern pattern = Pattern.compile("\\d+");
if (scriptResult != null && scriptResult.script != null) {
try {
if (scriptResult.script.htmlOutput != null) {
Matcher matcher = pattern.matcher(scriptResult.script.htmlOutput);
matcher.find();
String str = scriptResult.script.htmlOutput.substring(matcher.start(), matcher.end());
scriptResult.script.htmlOutput = scriptResult.script.htmlOutput.substring(matcher.end());
return Long.parseLong(str);
} else {
if (scriptResult.script.rawOutput != null) {
Matcher matcher = pattern.matcher(scriptResult.script.rawOutput);
matcher.find();
String str = scriptResult.script.rawOutput.substring(matcher.start(), matcher.end());
scriptResult.script.rawOutput = scriptResult.script.rawOutput.substring(matcher.end());
return Long.parseLong(str);
}
}
}
catch (Exception ex) {
return 1L;
}
}
return 1L;
}

}
7 changes: 5 additions & 2 deletions ...itrix3-monitor/src/main/java/dk/netarkivet/heritrix3/monitor/resources/H3JobResource.java
View file
Expand Up @@ -277,11 +277,14 @@ public void job(HttpServletRequest req, HttpServletResponse resp, List<Integer>
sb.append(thisAction);
sb.append("\"");
if("terminate".equals(thisAction) || "teardown".equals(thisAction)) {
sb.append("onclick=\"return confirm('Are you sure you wish to ");
sb.append(" onclick=\"return confirm('Are you sure you wish to ");
sb.append(thisAction);
sb.append(" the job currently being crawled ?')\"");
sb.append(" class=\"btn btn-danger\">");
sb.append("<i class=\\\"icon-white icon-trash\\\"></i>");
} else {
sb.append(" class=\"btn btn-default\">");
}
sb.append(" class=\"btn btn-default\">");
sb.append(job.availableActions.get(i).substring(0, 1).toUpperCase()+job.availableActions.get(i).substring(1));
sb.append("</a>");
}
Expand Down

0 comments on commit 0b21700

Please sign in to comment.