Skip to content
Permalink
Browse files

removed transformer

it was not used any more
  • Loading branch information...
Orbiter committed Jun 18, 2018
1 parent 495ca57 commit e0dc63202034266fc8442e9a52814155c7589d32
@@ -806,7 +806,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje
List<AnchorURL> hyperlinks_from_file;
// check if the crawl filter works correctly
final ContentScraper scraper = new ContentScraper(new DigestURL(crawlingFile), 10000000, new HashSet<String>(), new VocabularyScraper(), timezoneOffset);
final Writer writer = new TransformerWriter(null, null, scraper, null, false);
final Writer writer = new TransformerWriter(null, null, scraper, false);
if((crawlingFileContent == null || crawlingFileContent.isEmpty()) && crawlingFile != null) {
/* Let's report here detailed error to help user when he selected a wrong file */
if(!crawlingFile.exists()) {
@@ -143,7 +143,7 @@ public void run() {
this.profile, true);
this.scraper.registerHtmlFilterEventListener(anchorListener);

final Writer writer = new TransformerWriter(null, null, this.scraper, null, false);
final Writer writer = new TransformerWriter(null, null, this.scraper, false);
FileInputStream inStream = null;

try {
@@ -137,7 +137,7 @@ private static int importFromBookmarks(final BookmarksDB db, final DigestURL bas
//load the links
final ContentScraper scraper = new ContentScraper(baseURL, 10000, new HashSet<String>(), new VocabularyScraper(), 0);
//OutputStream os = new htmlFilterOutputStream(null, scraper, null, false);
final Writer writer = new TransformerWriter(null, null, scraper, null, false);
final Writer writer = new TransformerWriter(null, null, scraper, false);
FileUtils.copy(input,writer);
writer.close();
links = scraper.getAnchors();

This file was deleted.

@@ -1603,14 +1603,14 @@ public static ContentScraper parseResource(final File file, final int maxLinks,
if (page == null) throw new IOException("no content in file " + file.toString());

// scrape document to look up charset
final ScraperInputStream htmlFilter = new ScraperInputStream(new ByteArrayInputStream(page), StandardCharsets.UTF_8.name(), new HashSet<String>(), new VocabularyScraper(), new DigestURL("http://localhost"), null, false, maxLinks, timezoneOffset);
final ScraperInputStream htmlFilter = new ScraperInputStream(new ByteArrayInputStream(page), StandardCharsets.UTF_8.name(), new HashSet<String>(), new VocabularyScraper(), new DigestURL("http://localhost"), false, maxLinks, timezoneOffset);
String charset = htmlParser.patchCharsetEncoding(htmlFilter.detectCharset());
htmlFilter.close();
if (charset == null) charset = Charset.defaultCharset().toString();

// scrape content
final ContentScraper scraper = new ContentScraper(new DigestURL("http://localhost"), maxLinks, new HashSet<String>(), new VocabularyScraper(), timezoneOffset);
final Writer writer = new TransformerWriter(null, null, scraper, null, false);
final Writer writer = new TransformerWriter(null, null, scraper, false);
FileUtils.copy(new ByteArrayInputStream(page), writer, Charset.forName(charset));
writer.close();
return scraper;

This file was deleted.

@@ -65,7 +65,6 @@ public ScraperInputStream(
final Set<String> ignore_class_name,
final VocabularyScraper vocabularyScraper,
final DigestURL rooturl,
final Transformer transformer,
final boolean passbyIfBinarySuspect,
final int maxLinks,
final int timezoneOffset
@@ -82,7 +81,7 @@ public ScraperInputStream(
} catch (final UnsupportedEncodingException e) {
this.reader = new InputStreamReader(this, StandardCharsets.UTF_8);
}
this.writer = new TransformerWriter(null,null,scraper,transformer,passbyIfBinarySuspect);
this.writer = new TransformerWriter(null,null,scraper,passbyIfBinarySuspect);
}

private static String extractCharsetFromMimetypeHeader(final String mimeType) {

This file was deleted.

0 comments on commit e0dc632

Please sign in to comment.
You can’t perform that action at this time.