diff --git a/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java b/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java index 04a3a13030..6bfb6085fd 100644 --- a/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java +++ b/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java @@ -133,111 +133,111 @@ public void setConf(Configuration conf) { private void setHostSpecificIntervals(String fileName, float defaultMin, float defaultMax) throws IOException { // Setup for reading the config file. - Reader configReader = null; - configReader = conf.getConfResourceAsReader(fileName); + Reader configReader = conf.getConfResourceAsReader(fileName); if (configReader == null) { configReader = new FileReader(fileName, StandardCharsets.UTF_8); } - BufferedReader reader = new BufferedReader(configReader); - String line; - int lineNo = 0; + try (BufferedReader reader = new BufferedReader(configReader)) { + String line; + int lineNo = 0; - // Read the file line by line. - while ((line = reader.readLine()) != null) { - lineNo++; + // Read the file line by line. + while ((line = reader.readLine()) != null) { + lineNo++; - // Skip blank lines and comments. - if (StringUtils.isBlank(line) || line.startsWith("#")) { - continue; - } + // Skip blank lines and comments. + if (StringUtils.isBlank(line) || line.startsWith("#")) { + continue; + } - // Trim and partition the line. - line = line.trim(); - String[] parts = line.split("\\s+"); + // Trim and partition the line. + line = line.trim(); + String[] parts = line.split("\\s+"); - // There should be three parts. - if (parts.length != 3) { - LOG.error( - "Malformed (domain, min_interval, max_interval) triplet on line {} of the config. file: `{}`", - lineNo, line); - continue; - } + // There should be three parts. + if (parts.length != 3) { + LOG.error( + "Malformed (domain, min_interval, max_interval) triplet on line {} of the config. file: `{}`", + lineNo, line); + continue; + } - // Normalize the parts. - String host = parts[0].trim().toLowerCase(Locale.ROOT); - String minInt = parts[1].trim(); - String maxInt = parts[2].trim(); - - // "0" and "default" both mean `use default interval`; normalize to "0". - if (minInt.equalsIgnoreCase("default")) { minInt = "0"; } - if (maxInt.equalsIgnoreCase("default")) { maxInt = "0"; } - - // Convert intervals to float and ignore the line in case of failure. - float m, M; - try { - m = Float.parseFloat(minInt); - M = Float.parseFloat(maxInt); - } catch (NumberFormatException e) { - LOG.error( - "Improper fetch intervals given on line {} in the config. file `{}`: {}", - lineNo, line, e.toString()); - continue; - } + // Normalize the parts. + String host = parts[0].trim().toLowerCase(Locale.ROOT); + String minInt = parts[1].trim(); + String maxInt = parts[2].trim(); + + // "0" and "default" both mean `use default interval`; normalize to "0". + if (minInt.equalsIgnoreCase("default")) { minInt = "0"; } + if (maxInt.equalsIgnoreCase("default")) { maxInt = "0"; } + + // Convert intervals to float and ignore the line in case of failure. + float m, M; + try { + m = Float.parseFloat(minInt); + M = Float.parseFloat(maxInt); + } catch (NumberFormatException e) { + LOG.error( + "Improper fetch intervals given on line {} in the config. file `{}`: {}", + lineNo, line, e.toString()); + continue; + } - // If both intervals are set to default, - // ignore the line and issue a warning. - if (m == 0 && M == 0) { - LOG.warn( - "Ignoring default interval values on line {} of config. file: `{}`", - lineNo, line); - continue; - } + // If both intervals are set to default, + // ignore the line and issue a warning. + if (m == 0 && M == 0) { + LOG.warn( + "Ignoring default interval values on line {} of config. file: `{}`", + lineNo, line); + continue; + } - // Replace the zero with the default value. - if (m == 0) { - m = defaultMin; - } else if (M == 0) { - M = defaultMax; - } + // Replace the zero with the default value. + if (m == 0) { + m = defaultMin; + } else if (M == 0) { + M = defaultMax; + } - // Intervals cannot be negative and the min cannot be above the max - // (we assume here that the default values satisfy this). - if (m < 0 || M < 0) { - LOG.error( - "Improper fetch intervals given on line {} in the config. file: `{}`: intervals cannot be negative", - lineNo, line); - continue; - } + // Intervals cannot be negative and the min cannot be above the max + // (we assume here that the default values satisfy this). + if (m < 0 || M < 0) { + LOG.error( + "Improper fetch intervals given on line {} in the config. file: `{}`: intervals cannot be negative", + lineNo, line); + continue; + } - if (m > M) { - LOG.error( - "Improper fetch intervals given on line {} in the config. file: `{}`: min. interval cannot be above max. interval", - lineNo, line); - continue; - } + if (m > M) { + LOG.error( + "Improper fetch intervals given on line {} in the config. file: `{}`: min. interval cannot be above max. interval", + lineNo, line); + continue; + } - // The custom intervals should respect the boundaries of the default values. - if (m < defaultMin) { - LOG.error( - "Min. interval out of bounds ({}) on line {} in the config. file: `{}`", - defaultMin, lineNo, line); - continue; - } + // The custom intervals should respect the boundaries of the default values. + if (m < defaultMin) { + LOG.error( + "Min. interval out of bounds ({}) on line {} in the config. file: `{}`", + defaultMin, lineNo, line); + continue; + } - if (M > defaultMax) { - LOG.error( - "Max. interval out of bounds ({}) on line {} in the config. file: `{}`", - defaultMax, lineNo, line); - continue; - } + if (M > defaultMax) { + LOG.error( + "Max. interval out of bounds ({}) on line {} in the config. file: `{}`", + defaultMax, lineNo, line); + continue; + } - // If all is well, store the specific intervals. - hostSpecificMinInterval.put(host, m); - LOG.debug("Added custom min. interval {} for host {}.", m, host); + // If all is well, store the specific intervals. + hostSpecificMinInterval.put(host, m); + LOG.debug("Added custom min. interval {} for host {}.", m, host); - hostSpecificMaxInterval.put(host, M); - LOG.debug("Added custom max. interval {} for host {}.", M, host); + hostSpecificMaxInterval.put(host, M); + LOG.debug("Added custom max. interval {} for host {}.", M, host); + } } } diff --git a/src/java/org/apache/nutch/crawl/TextProfileSignature.java b/src/java/org/apache/nutch/crawl/TextProfileSignature.java index 63583a6941..f54d230ae6 100644 --- a/src/java/org/apache/nutch/crawl/TextProfileSignature.java +++ b/src/java/org/apache/nutch/crawl/TextProfileSignature.java @@ -195,17 +195,16 @@ public static void main(String[] args) throws Exception { HashMap res = new HashMap<>(); File[] files = new File(args[0]).listFiles(); for (int i = 0; i < files.length; i++) { - FileInputStream fis = new FileInputStream(files[i]); - BufferedReader br = new BufferedReader( - new InputStreamReader(fis, StandardCharsets.UTF_8)); StringBuffer text = new StringBuffer(); - String line = null; - while ((line = br.readLine()) != null) { - if (text.length() > 0) - text.append("\n"); - text.append(line); + try (BufferedReader br = new BufferedReader(new InputStreamReader( + new FileInputStream(files[i]), StandardCharsets.UTF_8))) { + String line = null; + while ((line = br.readLine()) != null) { + if (text.length() > 0) + text.append("\n"); + text.append(line); + } } - br.close(); byte[] signature = sig.calculate(null, new ParseImpl(text.toString(), null)); res.put(files[i].toString(), signature); diff --git a/src/java/org/apache/nutch/protocol/RobotRulesParser.java b/src/java/org/apache/nutch/protocol/RobotRulesParser.java index 5b0412910b..3efa0fc3b0 100644 --- a/src/java/org/apache/nutch/protocol/RobotRulesParser.java +++ b/src/java/org/apache/nutch/protocol/RobotRulesParser.java @@ -398,30 +398,30 @@ public int run(String[] args) { System.out.println("Testing robots.txt for agent names: " + (agentNames.isEmpty() ? "* (any other agent)" : agentNames)); - LineNumberReader testsIn = new LineNumberReader( - new FileReader(urlFile, StandardCharsets.UTF_8)); - String testPath; - testPath = testsIn.readLine(); - while (testPath != null) { - testPath = testPath.trim(); - try { - // testPath can be just a path or a complete URL - URL url = new URL(testPath); - String status; - if (isAllowListed(url)) { - status = "allowlisted"; - } else if (rules.isAllowed(testPath)) { - status = "allowed"; - } else { - status = "not allowed"; + try (LineNumberReader testsIn = new LineNumberReader( + new FileReader(urlFile, StandardCharsets.UTF_8))) { + String testPath; + testPath = testsIn.readLine(); + while (testPath != null) { + testPath = testPath.trim(); + try { + // testPath can be just a path or a complete URL + URL url = new URL(testPath); + String status; + if (isAllowListed(url)) { + status = "allowlisted"; + } else if (rules.isAllowed(testPath)) { + status = "allowed"; + } else { + status = "not allowed"; + } + System.out.println(status + ":\t" + testPath); + } catch (MalformedURLException e) { + LOG.warn("Not a valid URL: {}", testPath); } - System.out.println(status + ":\t" + testPath); - } catch (MalformedURLException e) { - LOG.warn("Not a valid URL: {}", testPath); + testPath = testsIn.readLine(); } - testPath = testsIn.readLine(); } - testsIn.close(); } catch (IOException e) { LOG.error("Failed to run:", e); return -1; @@ -476,9 +476,9 @@ public BaseRobotRules getRobotRulesSet(Protocol protocol, URL url, try { int contentLength = url.openConnection().getContentLength(); byte[] robotsBytes = new byte[contentLength]; - InputStream openStream = url.openStream(); - openStream.read(robotsBytes); - openStream.close(); + try (InputStream openStream = url.openStream()) { + openStream.read(robotsBytes); + } rules = robotParser.parseContent(url.toString(), robotsBytes, "text/plain", agentNames); } catch (IOException e) { diff --git a/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java b/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java index 80d6ac7cf5..de15e3d660 100644 --- a/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java +++ b/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java @@ -141,25 +141,26 @@ private int runCounter(FileSystem fs, Path webGraphDb) throws IOException, } Path numLinksFile = numLinksFiles[0].getPath(); LOG.info("Reading numlinks temp file {}", numLinksFile); - FSDataInputStream readLinks = fs.open(numLinksFile); CompressionCodecFactory cf = new CompressionCodecFactory(conf); CompressionCodec codec = cf.getCodec(numLinksFiles[0].getPath()); - InputStream streamLinks; - if (codec == null) { - LOG.debug("No compression codec found for {}, trying uncompressed", - numLinksFile); - streamLinks = readLinks; - } else { - LOG.info("Compression codec of numlinks temp file: {}", - codec.getDefaultExtension()); - readLinks.seek(0); - streamLinks = codec.createInputStream(readLinks); + String numLinksLine; + try (FSDataInputStream readLinks = fs.open(numLinksFile)) { + InputStream streamLinks; + if (codec == null) { + LOG.debug("No compression codec found for {}, trying uncompressed", + numLinksFile); + streamLinks = readLinks; + } else { + LOG.info("Compression codec of numlinks temp file: {}", + codec.getDefaultExtension()); + readLinks.seek(0); + streamLinks = codec.createInputStream(readLinks); + } + try (BufferedReader buffer = new BufferedReader( + new InputStreamReader(streamLinks, StandardCharsets.UTF_8))) { + numLinksLine = buffer.readLine(); + } } - BufferedReader buffer = new BufferedReader( - new InputStreamReader(streamLinks, StandardCharsets.UTF_8)); - - String numLinksLine = buffer.readLine(); - readLinks.close(); // check if there are links to process, if none, webgraph might be empty if (numLinksLine == null || numLinksLine.length() == 0) { diff --git a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java index 39935016da..fe714218e9 100644 --- a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java +++ b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java @@ -269,182 +269,180 @@ public void dump(File outputDir, File segmentRootDir, File linkdb, boolean gzip, } } - LinkDbReader linkDbReader = null; - if (linkdb != null) { - linkDbReader = new LinkDbReader(nutchConfig, new Path(linkdb.toString())); - } - if (parts == null || parts.size() == 0) { - LOG.error( "No segment directories found in {} ", - segmentRootDir.getAbsolutePath()); - this.errorTracker.recordError(ErrorTracker.ErrorType.OTHER); - return; - } - LOG.info("Found {} segment parts", parts.size()); - if (gzip && !warc) { - fileList = new ArrayList<>(); - constructNewStream(outputDir); - } + try (LinkDbReader linkDbReader = linkdb != null + ? new LinkDbReader(nutchConfig, new Path(linkdb.toString())) + : null) { + if (parts == null || parts.size() == 0) { + LOG.error( "No segment directories found in {} ", + segmentRootDir.getAbsolutePath()); + this.errorTracker.recordError(ErrorTracker.ErrorType.OTHER); + return; + } + LOG.info("Found {} segment parts", parts.size()); + if (gzip && !warc) { + fileList = new ArrayList<>(); + constructNewStream(outputDir); + } - for (Path segmentPart : parts) { - LOG.info("Processing segment Part : [ {} ]", segmentPart); - try { - SequenceFile.Reader reader = new SequenceFile.Reader(nutchConfig, - SequenceFile.Reader.file(segmentPart)); + for (Path segmentPart : parts) { + LOG.info("Processing segment Part : [ {} ]", segmentPart); + try (SequenceFile.Reader reader = new SequenceFile.Reader(nutchConfig, + SequenceFile.Reader.file(segmentPart))) { - Writable key = (Writable) reader.getKeyClass().getConstructor().newInstance(); + Writable key = (Writable) reader.getKeyClass().getConstructor().newInstance(); - Content content = null; - while (reader.next(key)) { - content = new Content(); - reader.getCurrentValue(content); - Metadata metadata = content.getMetadata(); - String url = key.toString(); + Content content = null; + while (reader.next(key)) { + content = new Content(); + reader.getCurrentValue(content); + Metadata metadata = content.getMetadata(); + String url = key.toString(); - String baseName = FilenameUtils.getBaseName(url); - String extensionName = FilenameUtils.getExtension(url); + String baseName = FilenameUtils.getBaseName(url); + String extensionName = FilenameUtils.getExtension(url); - if (!extension.isEmpty()) { - extensionName = extension; - } else if ((extensionName == null) || extensionName.isEmpty()) { - extensionName = "html"; - } + if (!extension.isEmpty()) { + extensionName = extension; + } else if ((extensionName == null) || extensionName.isEmpty()) { + extensionName = "html"; + } - String outputFullPath = null; - String outputRelativePath = null; - String filename = null; - String timestamp = null; - String reverseKey = null; + String outputFullPath = null; + String outputRelativePath = null; + String filename = null; + String timestamp = null; + String reverseKey = null; + + if (epochFilename || config.getReverseKey()) { + try { + long epoch = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z") + .parse(getDate(metadata.get("Date"))).getTime(); + timestamp = String.valueOf(epoch); + } catch (ParseException pe) { + LOG.warn(pe.getMessage()); + } - if (epochFilename || config.getReverseKey()) { - try { - long epoch = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z") - .parse(getDate(metadata.get("Date"))).getTime(); - timestamp = String.valueOf(epoch); - } catch (ParseException pe) { - LOG.warn(pe.getMessage()); + reverseKey = reverseUrl(url); + config.setReverseKeyValue( + reverseKey.replace("/", "_") + "_" + DigestUtils.sha1Hex(url) + + "_" + timestamp); } - reverseKey = reverseUrl(url); - config.setReverseKeyValue( - reverseKey.replace("/", "_") + "_" + DigestUtils.sha1Hex(url) - + "_" + timestamp); - } - - if (!warc) { - if (epochFilename) { - outputFullPath = DumpFileUtil - .createFileNameFromUrl(outputDir.getAbsolutePath(), - reverseKey, url, timestamp, extensionName, !gzip); - outputRelativePath = outputFullPath - .substring(0, outputFullPath.lastIndexOf(File.separator) - 1); - filename = content.getMetadata().get(Metadata.DATE) + "." - + extensionName; - } else { - String md5Ofurl = DumpFileUtil.getUrlMD5(url); - String fullDir = DumpFileUtil - .createTwoLevelsDirectory(outputDir.getAbsolutePath(), - md5Ofurl, !gzip); - filename = DumpFileUtil - .createFileName(md5Ofurl, baseName, extensionName); - outputFullPath = String.format(Locale.ROOT, "%s/%s", fullDir, filename); - - String[] fullPathLevels = fullDir - .split(Pattern.quote(File.separator)); - String firstLevelDirName = fullPathLevels[fullPathLevels.length - - 2]; - String secondLevelDirName = fullPathLevels[fullPathLevels.length - - 1]; - outputRelativePath = firstLevelDirName + secondLevelDirName; + if (!warc) { + if (epochFilename) { + outputFullPath = DumpFileUtil + .createFileNameFromUrl(outputDir.getAbsolutePath(), + reverseKey, url, timestamp, extensionName, !gzip); + outputRelativePath = outputFullPath + .substring(0, outputFullPath.lastIndexOf(File.separator) - 1); + filename = content.getMetadata().get(Metadata.DATE) + "." + + extensionName; + } else { + String md5Ofurl = DumpFileUtil.getUrlMD5(url); + String fullDir = DumpFileUtil + .createTwoLevelsDirectory(outputDir.getAbsolutePath(), + md5Ofurl, !gzip); + filename = DumpFileUtil + .createFileName(md5Ofurl, baseName, extensionName); + outputFullPath = String.format(Locale.ROOT, "%s/%s", fullDir, filename); + + String[] fullPathLevels = fullDir + .split(Pattern.quote(File.separator)); + String firstLevelDirName = fullPathLevels[fullPathLevels.length + - 2]; + String secondLevelDirName = fullPathLevels[fullPathLevels.length + - 1]; + outputRelativePath = firstLevelDirName + secondLevelDirName; + } } - } - // Encode all filetypes if no mimetypes have been given - Boolean filter = (mimeTypes == null); - - String jsonData = ""; - try { - String mimeType = new Tika().detect(content.getContent()); - // Maps file to JSON-based structure - - Set inUrls = null; //there may be duplicates, so using set - if (linkDbReader != null) { - Inlinks inlinks = linkDbReader.getInlinks((Text) key); - if (inlinks != null) { - Iterator iterator = inlinks.iterator(); - inUrls = new LinkedHashSet<>(); - while (inUrls.size() <= MAX_INLINKS && iterator.hasNext()){ - inUrls.add(iterator.next().getFromUrl()); + // Encode all filetypes if no mimetypes have been given + Boolean filter = (mimeTypes == null); + + String jsonData = ""; + try { + String mimeType = new Tika().detect(content.getContent()); + // Maps file to JSON-based structure + + Set inUrls = null; //there may be duplicates, so using set + if (linkDbReader != null) { + Inlinks inlinks = linkDbReader.getInlinks((Text) key); + if (inlinks != null) { + Iterator iterator = inlinks.iterator(); + inUrls = new LinkedHashSet<>(); + while (inUrls.size() <= MAX_INLINKS && iterator.hasNext()){ + inUrls.add(iterator.next().getFromUrl()); + } } } - } - //TODO: Make this Jackson Format implementation reusable - try (CommonCrawlFormat format = CommonCrawlFormatFactory - .getCommonCrawlFormat(warc ? "WARC" : "JACKSON", nutchConfig, config)) { - if (inUrls != null) { - format.setInLinks(new ArrayList<>(inUrls)); + //TODO: Make this Jackson Format implementation reusable + try (CommonCrawlFormat format = CommonCrawlFormatFactory + .getCommonCrawlFormat(warc ? "WARC" : "JACKSON", nutchConfig, config)) { + if (inUrls != null) { + format.setInLinks(new ArrayList<>(inUrls)); + } + jsonData = format.getJsonData(url, content, metadata); } - jsonData = format.getJsonData(url, content, metadata); - } - collectStats(typeCounts, mimeType); - // collects statistics for the given mimetypes - if ((mimeType != null) && (mimeTypes != null) && Arrays - .asList(mimeTypes).contains(mimeType)) { - collectStats(filteredCounts, mimeType); - filter = true; + collectStats(typeCounts, mimeType); + // collects statistics for the given mimetypes + if ((mimeType != null) && (mimeTypes != null) && Arrays + .asList(mimeTypes).contains(mimeType)) { + collectStats(filteredCounts, mimeType); + filter = true; + } + } catch (IOException ioe) { + LOG.error("Fatal error in creating JSON data: {}", ioe.getMessage()); + return; } - } catch (IOException ioe) { - LOG.error("Fatal error in creating JSON data: {}", ioe.getMessage()); - return; - } - if (!warc) { - if (filter) { - byte[] byteData = serializeCBORData(jsonData); - - if (!gzip) { - File outputFile = new File(outputFullPath); - if (outputFile.exists()) { - LOG.info("Skipping writing: [{}]: file already exists", outputFullPath); - } else { - LOG.info("Writing: [{}]", outputFullPath); - IOUtils.copy(new ByteArrayInputStream(byteData), - new FileOutputStream(outputFile)); - } - } else { - if (fileList.contains(outputFullPath)) { - LOG.info("Skipping compressing: [{}]: file already exists", outputFullPath); + if (!warc) { + if (filter) { + byte[] byteData = serializeCBORData(jsonData); + + if (!gzip) { + File outputFile = new File(outputFullPath); + if (outputFile.exists()) { + LOG.info("Skipping writing: [{}]: file already exists", outputFullPath); + } else { + LOG.info("Writing: [{}]", outputFullPath); + IOUtils.copy(new ByteArrayInputStream(byteData), + new FileOutputStream(outputFile)); + } } else { - fileList.add(outputFullPath); - LOG.info("Compressing: [{}]", outputFullPath); - //TarArchiveEntry tarEntry = new TarArchiveEntry(firstLevelDirName + File.separator + secondLevelDirName + File.separator + filename); - TarArchiveEntry tarEntry = new TarArchiveEntry( - outputRelativePath + File.separator + filename); - tarEntry.setSize(byteData.length); - tarOutput.putArchiveEntry(tarEntry); - tarOutput.write(byteData); - tarOutput.closeArchiveEntry(); + if (fileList.contains(outputFullPath)) { + LOG.info("Skipping compressing: [{}]: file already exists", outputFullPath); + } else { + fileList.add(outputFullPath); + LOG.info("Compressing: [{}]", outputFullPath); + //TarArchiveEntry tarEntry = new TarArchiveEntry(firstLevelDirName + File.separator + secondLevelDirName + File.separator + filename); + TarArchiveEntry tarEntry = new TarArchiveEntry( + outputRelativePath + File.separator + filename); + tarEntry.setSize(byteData.length); + tarOutput.putArchiveEntry(tarEntry); + tarOutput.write(byteData); + tarOutput.closeArchiveEntry(); + } } } } } + } catch (Exception e){ + LOG.warn("SKIPPED: {} Because : {}", segmentPart, e.getMessage()); + } finally { + fs.close(); } - reader.close(); - } catch (Exception e){ - LOG.warn("SKIPPED: {} Because : {}", segmentPart, e.getMessage()); - } finally { - fs.close(); } - } - if (gzip && !warc) { - closeStream(); - } + if (gzip && !warc) { + closeStream(); + } - if (!typeCounts.isEmpty()) { - LOG.info("CommonsCrawlDataDumper File Stats: {}", DumpFileUtil - .displayFileTypes(typeCounts, filteredCounts)); - } + if (!typeCounts.isEmpty()) { + LOG.info("CommonsCrawlDataDumper File Stats: {}", DumpFileUtil + .displayFileTypes(typeCounts, filteredCounts)); + } + } } private void closeStream() { diff --git a/src/java/org/apache/nutch/tools/FileDumper.java b/src/java/org/apache/nutch/tools/FileDumper.java index 8afc390c6d..35d096351b 100644 --- a/src/java/org/apache/nutch/tools/FileDumper.java +++ b/src/java/org/apache/nutch/tools/FileDumper.java @@ -169,114 +169,107 @@ public void dump(File outputDir, File segmentRootDir, String[] mimeTypes, boolea continue; } - SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(file)); - - Writable key = (Writable) reader.getKeyClass().getConstructor().newInstance(); - Content content = null; - - while (reader.next(key)) { - content = new Content(); - reader.getCurrentValue(content); - String url = key.toString(); - String baseName = FilenameUtils.getBaseName(url); - String extension = FilenameUtils.getExtension(url); - if (extension == null || (extension != null && extension.equals(""))) { - extension = "html"; - } + try (SequenceFile.Reader reader = new SequenceFile.Reader(conf, + SequenceFile.Reader.file(file))) { + + Writable key = (Writable) reader.getKeyClass().getConstructor().newInstance(); + Content content = null; + + while (reader.next(key)) { + content = new Content(); + reader.getCurrentValue(content); + String url = key.toString(); + String baseName = FilenameUtils.getBaseName(url); + String extension = FilenameUtils.getExtension(url); + if (extension == null || (extension != null && extension.equals(""))) { + extension = "html"; + } - ByteArrayInputStream bas = null; - Boolean filter = false; - try { - bas = new ByteArrayInputStream(content.getContent()); - String mimeType = new Tika().detect(content.getContent()); - collectStats(typeCounts, mimeType); - if (mimeType != null) { - if (mimeTypes == null - || Arrays.asList(mimeTypes).contains(mimeType)) { - collectStats(filteredCounts, mimeType); - filter = true; + ByteArrayInputStream bas = null; + Boolean filter = false; + try { + bas = new ByteArrayInputStream(content.getContent()); + String mimeType = new Tika().detect(content.getContent()); + collectStats(typeCounts, mimeType); + if (mimeType != null) { + if (mimeTypes == null + || Arrays.asList(mimeTypes).contains(mimeType)) { + collectStats(filteredCounts, mimeType); + filter = true; + } } - } - } catch (Exception e) { - e.printStackTrace(); - LOG.warn("Tika is unable to detect type for: [{}]", url); - } finally { - if (bas != null) { - try { - bas.close(); - } catch (Exception ignore) { + } catch (Exception e) { + e.printStackTrace(); + LOG.warn("Tika is unable to detect type for: [{}]", url); + } finally { + if (bas != null) { + try { + bas.close(); + } catch (Exception ignore) { + } } } - } - if (filter) { - if (!mimeTypeStats) { - String md5Ofurl = DumpFileUtil.getUrlMD5(url); + if (filter) { + if (!mimeTypeStats) { + String md5Ofurl = DumpFileUtil.getUrlMD5(url); - String fullDir = outputDir.getAbsolutePath(); - if (!flatDir && !reverseURLDump) { - fullDir = DumpFileUtil.createTwoLevelsDirectory(fullDir, md5Ofurl); - } + String fullDir = outputDir.getAbsolutePath(); + if (!flatDir && !reverseURLDump) { + fullDir = DumpFileUtil.createTwoLevelsDirectory(fullDir, md5Ofurl); + } - if (!Strings.isNullOrEmpty(fullDir)) { - String outputFullPath; + if (!Strings.isNullOrEmpty(fullDir)) { + String outputFullPath; - if (reverseURLDump) { - String[] reversedURL = TableUtil.reverseUrl(url).split(":"); - reversedURL[0] = reversedURL[0].replace('.', '/'); + if (reverseURLDump) { + String[] reversedURL = TableUtil.reverseUrl(url).split(":"); + reversedURL[0] = reversedURL[0].replace('.', '/'); - String reversedURLPath = reversedURL[0] + "/" - + DigestUtils.sha256Hex(url).toUpperCase(Locale.ROOT); - outputFullPath = String.format(Locale.ROOT, "%s/%s", - fullDir, reversedURLPath); + String reversedURLPath = reversedURL[0] + "/" + + DigestUtils.sha256Hex(url).toUpperCase(Locale.ROOT); + outputFullPath = String.format(Locale.ROOT, "%s/%s", + fullDir, reversedURLPath); - // We'll drop the trailing file name and create the nested structure if it doesn't already exist. - String[] splitPath = outputFullPath.split("/"); - File fullOutputDir = new File(org.apache.commons.lang3.StringUtils.join(Arrays.copyOf(splitPath, splitPath.length - 1), "/")); + // We'll drop the trailing file name and create the nested structure if it doesn't already exist. + String[] splitPath = outputFullPath.split("/"); + File fullOutputDir = new File(org.apache.commons.lang3.StringUtils.join(Arrays.copyOf(splitPath, splitPath.length - 1), "/")); - if (!fullOutputDir.exists()) { - if(!fullOutputDir.mkdirs()); - throw new Exception("Unable to create: [" - + fullOutputDir.getAbsolutePath() + "]"); + if (!fullOutputDir.exists()) { + if(!fullOutputDir.mkdirs()); + throw new Exception("Unable to create: [" + + fullOutputDir.getAbsolutePath() + "]"); + } + } else { + outputFullPath = String.format(Locale.ROOT, "%s/%s", + fullDir, DumpFileUtil.createFileName(md5Ofurl, baseName, + extension)); } - } else { - outputFullPath = String.format(Locale.ROOT, "%s/%s", - fullDir, DumpFileUtil.createFileName(md5Ofurl, baseName, - extension)); - } - filenameToUrl.put(outputFullPath, url); - File outputFile = new File(outputFullPath); - - if (!outputFile.exists()) { - LOG.info("Writing: [{}]", outputFullPath); - - // Modified to prevent FileNotFoundException (Invalid Argument) - FileOutputStream output = null; - try { - output = new FileOutputStream(outputFile); - IOUtils.write(content.getContent(), output); - } catch (Exception e) { - LOG.warn("Write Error: [{}]", outputFullPath); - e.printStackTrace(); - } finally { - if (output != null) { + filenameToUrl.put(outputFullPath, url); + File outputFile = new File(outputFullPath); + + if (!outputFile.exists()) { + LOG.info("Writing: [{}]", outputFullPath); + + // Modified to prevent FileNotFoundException (Invalid Argument) + try (FileOutputStream output = new FileOutputStream( + outputFile)) { + IOUtils.write(content.getContent(), output); output.flush(); - try { - output.close(); - } catch (Exception ignore) { - } + } catch (Exception e) { + LOG.warn("Write Error: [{}]", outputFullPath); + e.printStackTrace(); } + fileCount++; + } else { + LOG.info("Skipping writing: [{}]: file already exists", + outputFullPath); } - fileCount++; - } else { - LOG.info("Skipping writing: [{}]: file already exists", - outputFullPath); } } } } } - reader.close(); } finally { if (doutputStream != null) { try { diff --git a/src/java/org/apache/nutch/tools/ResolveUrls.java b/src/java/org/apache/nutch/tools/ResolveUrls.java index ad38307a60..ddfa6cf219 100644 --- a/src/java/org/apache/nutch/tools/ResolveUrls.java +++ b/src/java/org/apache/nutch/tools/ResolveUrls.java @@ -107,19 +107,19 @@ public void resolveUrls() { pool = Executors.newFixedThreadPool(numThreads); // read in the urls file and loop through each line, one url per line - BufferedReader buffRead = new BufferedReader(new FileReader(new File( - urlsFile), StandardCharsets.UTF_8)); - String urlStr = null; - while ((urlStr = buffRead.readLine()) != null) { - - // spin up a resolver thread per url - LOG.info("Starting: {}", urlStr); - pool.execute(new ResolverThread(urlStr)); + try (BufferedReader buffRead = new BufferedReader(new FileReader( + new File(urlsFile), StandardCharsets.UTF_8))) { + String urlStr = null; + while ((urlStr = buffRead.readLine()) != null) { + + // spin up a resolver thread per url + LOG.info("Starting: {}", urlStr); + pool.execute(new ResolverThread(urlStr)); + } } - // close the file and wait for up to 60 seconds before shutting down - // the thread pool to give urls time to finish resolving - buffRead.close(); + // wait for up to 60 seconds before shutting down the thread pool to + // give urls time to finish resolving pool.awaitTermination(60, TimeUnit.SECONDS); } catch (Exception e) { diff --git a/src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.java b/src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.java index a444f43121..720f5a6d1e 100644 --- a/src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.java +++ b/src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.java @@ -198,9 +198,9 @@ public static void main(String[] args) throws Exception { parser.setConf(conf); File file = new File(name); byte[] bytes = new byte[(int) file.length()]; - DataInputStream in = new DataInputStream(new FileInputStream(file)); - in.readFully(bytes); - in.close(); + try (DataInputStream in = new DataInputStream(new FileInputStream(file))) { + in.readFully(bytes); + } ParseResult parseResult = parser.getParse(new Content(url, url, bytes, "application/rss+xml", new Metadata(), conf)); for (Entry entry : parseResult) { diff --git a/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java b/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java index 172b2030af..8e83572f31 100644 --- a/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java +++ b/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java @@ -18,7 +18,6 @@ import java.io.BufferedReader; import java.io.FileInputStream; -import java.io.InputStream; import java.io.InputStreamReader; import java.lang.invoke.MethodHandles; import java.net.MalformedURLException; @@ -278,14 +277,14 @@ public static void main(String[] args) throws Exception { System.err.println(JSParseFilter.class.getName() + " file.js baseURL"); return; } - InputStream in = new FileInputStream(args[0]); - BufferedReader br = new BufferedReader( - new InputStreamReader(in, StandardCharsets.UTF_8)); StringBuffer sb = new StringBuffer(); - String line = null; - while ((line = br.readLine()) != null) - sb.append(line + "\n"); - br.close(); + try (BufferedReader br = new BufferedReader( + new InputStreamReader(new FileInputStream(args[0]), + StandardCharsets.UTF_8))) { + String line = null; + while ((line = br.readLine()) != null) + sb.append(line + "\n"); + } JSParseFilter parseFilter = new JSParseFilter(); parseFilter.setConf(NutchConfiguration.create()); diff --git a/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java b/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java index 315f7e79b6..7dc08fd7b6 100644 --- a/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java +++ b/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java @@ -124,10 +124,11 @@ public static void main(String[] args) throws IOException { } File file = new File(args[0]); String url = "file:"+file.getCanonicalPath(); - FileInputStream in = new FileInputStream(file); - byte[] bytes = new byte[in.available()]; - in.read(bytes); - in.close(); + byte[] bytes; + try (FileInputStream in = new FileInputStream(file)) { + bytes = new byte[in.available()]; + in.read(bytes); + } Configuration conf = NutchConfiguration.create(); ZipParser parser = new ZipParser(); parser.setConf(conf); diff --git a/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Classify.java b/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Classify.java index 4decb7a358..362c341cc6 100644 --- a/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Classify.java +++ b/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Classify.java @@ -70,23 +70,23 @@ public static String classify(String line) throws IOException { Configuration configuration = new Configuration(); FileSystem fs = FileSystem.get(configuration); - BufferedReader bufferedReader = new BufferedReader(new InputStreamReader( - fs.open(new Path("naivebayes-model")), StandardCharsets.UTF_8)); - - uniquewords_size = Integer.parseInt(bufferedReader.readLine()); - bufferedReader.readLine(); - - numof_ir = Integer.parseInt(bufferedReader.readLine()); - numwords_ir = Integer.parseInt(bufferedReader.readLine()); - wordfreq_ir = unflattenToHashmap(bufferedReader.readLine()); - bufferedReader.readLine(); - numof_r = Integer.parseInt(bufferedReader.readLine()); - numwords_r = Integer.parseInt(bufferedReader.readLine()); - wordfreq_r = unflattenToHashmap(bufferedReader.readLine()); - - ismodel = true; - - bufferedReader.close(); + try (BufferedReader bufferedReader = new BufferedReader( + new InputStreamReader(fs.open(new Path("naivebayes-model")), + StandardCharsets.UTF_8))) { + + uniquewords_size = Integer.parseInt(bufferedReader.readLine()); + bufferedReader.readLine(); + + numof_ir = Integer.parseInt(bufferedReader.readLine()); + numwords_ir = Integer.parseInt(bufferedReader.readLine()); + wordfreq_ir = unflattenToHashmap(bufferedReader.readLine()); + bufferedReader.readLine(); + numof_r = Integer.parseInt(bufferedReader.readLine()); + numwords_r = Integer.parseInt(bufferedReader.readLine()); + wordfreq_r = unflattenToHashmap(bufferedReader.readLine()); + + ismodel = true; + } } diff --git a/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Train.java b/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Train.java index ab021334b5..0b23705f44 100644 --- a/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Train.java +++ b/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Train.java @@ -90,34 +90,36 @@ public static void start(String filepath) throws IOException { Configuration configuration = new Configuration(); FileSystem fs = FileSystem.get(configuration); - BufferedReader bufferedReader = new BufferedReader( - configuration.getConfResourceAsReader(filepath)); + try (BufferedReader bufferedReader = new BufferedReader( + configuration.getConfResourceAsReader(filepath))) { - while ((line = bufferedReader.readLine()) != null) { + while ((line = bufferedReader.readLine()) != null) { - target = line.split("\t")[0]; + target = line.split("\t")[0]; - line = replacefirstoccuranceof(target + "\t", line); + line = replacefirstoccuranceof(target + "\t", line); - linearray = line.replaceAll("[^a-zA-Z ]", "").toLowerCase(Locale.ROOT) - .split(" "); + linearray = line.replaceAll("[^a-zA-Z ]", "").toLowerCase(Locale.ROOT) + .split(" "); - // update the data structures - if (target.equals("0")) { + // update the data structures + if (target.equals("0")) { - numof_ir += 1; - numwords_ir += linearray.length; - for (int i = 0; i < linearray.length; i++) { - uniquewords.add(linearray[i]); - updateHashMap(wordfreq_ir, linearray[i]); - } - } else { + numof_ir += 1; + numwords_ir += linearray.length; + for (int i = 0; i < linearray.length; i++) { + uniquewords.add(linearray[i]); + updateHashMap(wordfreq_ir, linearray[i]); + } + } else { + + numof_r += 1; + numwords_r += linearray.length; + for (int i = 0; i < linearray.length; i++) { + uniquewords.add(linearray[i]); + updateHashMap(wordfreq_r, linearray[i]); + } - numof_r += 1; - numwords_r += linearray.length; - for (int i = 0; i < linearray.length; i++) { - uniquewords.add(linearray[i]); - updateHashMap(wordfreq_r, linearray[i]); } } @@ -128,22 +130,19 @@ public static void start(String filepath) throws IOException { Path path = new Path("naivebayes-model"); - Writer writer = new BufferedWriter(new OutputStreamWriter(fs.create(path, - true), StandardCharsets.UTF_8)); - - writer.write(String.valueOf(uniquewords.size()) + "\n"); - writer.write("0\n"); - writer.write(String.valueOf(numof_ir) + "\n"); - writer.write(String.valueOf(numwords_ir) + "\n"); - writer.write(flattenHashMap(wordfreq_ir) + "\n"); - writer.write("1\n"); - writer.write(String.valueOf(numof_r) + "\n"); - writer.write(String.valueOf(numwords_r) + "\n"); - writer.write(flattenHashMap(wordfreq_r) + "\n"); - - writer.close(); - - bufferedReader.close(); + try (Writer writer = new BufferedWriter(new OutputStreamWriter(fs.create( + path, true), StandardCharsets.UTF_8))) { + + writer.write(String.valueOf(uniquewords.size()) + "\n"); + writer.write("0\n"); + writer.write(String.valueOf(numof_ir) + "\n"); + writer.write(String.valueOf(numwords_ir) + "\n"); + writer.write(flattenHashMap(wordfreq_ir) + "\n"); + writer.write("1\n"); + writer.write(String.valueOf(numof_r) + "\n"); + writer.write(String.valueOf(numwords_r) + "\n"); + writer.write(flattenHashMap(wordfreq_r) + "\n"); + } } diff --git a/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java b/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java index 266aab165b..143ef3d1e5 100644 --- a/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java +++ b/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java @@ -204,17 +204,17 @@ private void getFileAsHttpResponse(java.io.File f) throws FileException, this.content = new byte[len]; - java.io.InputStream is = new java.io.FileInputStream(f); - int offset = 0; - int n = 0; - while (offset < len - && (n = is.read(this.content, offset, len - offset)) >= 0) { - offset += n; - } - if (offset < len) { // keep whatever already have, but issue a warning - File.LOG.warn("not enough bytes read from file: {}", f.getPath()); + try (java.io.InputStream is = new java.io.FileInputStream(f)) { + int offset = 0; + int n = 0; + while (offset < len + && (n = is.read(this.content, offset, len - offset)) >= 0) { + offset += n; + } + if (offset < len) { // keep whatever already have, but issue a warning + File.LOG.warn("not enough bytes read from file: {}", f.getPath()); + } } - is.close(); // set headers headers.set(Response.CONTENT_LENGTH, Long.valueOf(size).toString());