Skip to content

NUTCH-3200 Close Closeable resources flagged by Sonar java:S2095 #948

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lewismc wants to merge 2 commits into
base: master
Choose a base branch
from
+455 −465
Open

NUTCH-3200 Close Closeable resources flagged by Sonar java:S2095 #948

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4126dcd
NUTCH-3200 Close Closeable resources flagged by Sonar java:S2095
lewismc Aug 5, 2026
6937bcf
NUTCH-3200 Close Closeable resources flagged by Sonar java:S2095
lewismc Aug 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 88 additions & 88 deletions src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,111 +133,111 @@ public void setConf(Configuration conf) {
private void setHostSpecificIntervals(String fileName,
float defaultMin, float defaultMax) throws IOException {
// Setup for reading the config file.
Reader configReader = null;
configReader = conf.getConfResourceAsReader(fileName);
Reader configReader = conf.getConfResourceAsReader(fileName);
if (configReader == null) {
configReader = new FileReader(fileName, StandardCharsets.UTF_8);
}
BufferedReader reader = new BufferedReader(configReader);
String line;
int lineNo = 0;
try (BufferedReader reader = new BufferedReader(configReader)) {
String line;
int lineNo = 0;

// Read the file line by line.
while ((line = reader.readLine()) != null) {
lineNo++;
// Read the file line by line.
while ((line = reader.readLine()) != null) {
lineNo++;

// Skip blank lines and comments.
if (StringUtils.isBlank(line) || line.startsWith("#")) {
continue;
}
// Skip blank lines and comments.
if (StringUtils.isBlank(line) || line.startsWith("#")) {
continue;
}

// Trim and partition the line.
line = line.trim();
String[] parts = line.split("\\s+");
// Trim and partition the line.
line = line.trim();
String[] parts = line.split("\\s+");

// There should be three parts.
if (parts.length != 3) {
LOG.error(
"Malformed (domain, min_interval, max_interval) triplet on line {} of the config. file: `{}`",
lineNo, line);
continue;
}
// There should be three parts.
if (parts.length != 3) {
LOG.error(
"Malformed (domain, min_interval, max_interval) triplet on line {} of the config. file: `{}`",
lineNo, line);
continue;
}

// Normalize the parts.
String host = parts[0].trim().toLowerCase(Locale.ROOT);
String minInt = parts[1].trim();
String maxInt = parts[2].trim();

// "0" and "default" both mean `use default interval`; normalize to "0".
if (minInt.equalsIgnoreCase("default")) { minInt = "0"; }
if (maxInt.equalsIgnoreCase("default")) { maxInt = "0"; }

// Convert intervals to float and ignore the line in case of failure.
float m, M;
try {
m = Float.parseFloat(minInt);
M = Float.parseFloat(maxInt);
} catch (NumberFormatException e) {
LOG.error(
"Improper fetch intervals given on line {} in the config. file `{}`: {}",
lineNo, line, e.toString());
continue;
}
// Normalize the parts.
String host = parts[0].trim().toLowerCase(Locale.ROOT);
String minInt = parts[1].trim();
String maxInt = parts[2].trim();

// "0" and "default" both mean `use default interval`; normalize to "0".
if (minInt.equalsIgnoreCase("default")) { minInt = "0"; }
if (maxInt.equalsIgnoreCase("default")) { maxInt = "0"; }

// Convert intervals to float and ignore the line in case of failure.
float m, M;
try {
m = Float.parseFloat(minInt);
M = Float.parseFloat(maxInt);
} catch (NumberFormatException e) {
LOG.error(
"Improper fetch intervals given on line {} in the config. file `{}`: {}",
lineNo, line, e.toString());
continue;
}

// If both intervals are set to default,
// ignore the line and issue a warning.
if (m == 0 && M == 0) {
LOG.warn(
"Ignoring default interval values on line {} of config. file: `{}`",
lineNo, line);
continue;
}
// If both intervals are set to default,
// ignore the line and issue a warning.
if (m == 0 && M == 0) {
LOG.warn(
"Ignoring default interval values on line {} of config. file: `{}`",
lineNo, line);
continue;
}

// Replace the zero with the default value.
if (m == 0) {
m = defaultMin;
} else if (M == 0) {
M = defaultMax;
}
// Replace the zero with the default value.
if (m == 0) {
m = defaultMin;
} else if (M == 0) {
M = defaultMax;
}

// Intervals cannot be negative and the min cannot be above the max
// (we assume here that the default values satisfy this).
if (m < 0 || M < 0) {
LOG.error(
"Improper fetch intervals given on line {} in the config. file: `{}`: intervals cannot be negative",
lineNo, line);
continue;
}
// Intervals cannot be negative and the min cannot be above the max
// (we assume here that the default values satisfy this).
if (m < 0 || M < 0) {
LOG.error(
"Improper fetch intervals given on line {} in the config. file: `{}`: intervals cannot be negative",
lineNo, line);
continue;
}

if (m > M) {
LOG.error(
"Improper fetch intervals given on line {} in the config. file: `{}`: min. interval cannot be above max. interval",
lineNo, line);
continue;
}
if (m > M) {
LOG.error(
"Improper fetch intervals given on line {} in the config. file: `{}`: min. interval cannot be above max. interval",
lineNo, line);
continue;
}

// The custom intervals should respect the boundaries of the default values.
if (m < defaultMin) {
LOG.error(
"Min. interval out of bounds ({}) on line {} in the config. file: `{}`",
defaultMin, lineNo, line);
continue;
}
// The custom intervals should respect the boundaries of the default values.
if (m < defaultMin) {
LOG.error(
"Min. interval out of bounds ({}) on line {} in the config. file: `{}`",
defaultMin, lineNo, line);
continue;
}

if (M > defaultMax) {
LOG.error(
"Max. interval out of bounds ({}) on line {} in the config. file: `{}`",
defaultMax, lineNo, line);
continue;
}
if (M > defaultMax) {
LOG.error(
"Max. interval out of bounds ({}) on line {} in the config. file: `{}`",
defaultMax, lineNo, line);
continue;
}

// If all is well, store the specific intervals.
hostSpecificMinInterval.put(host, m);
LOG.debug("Added custom min. interval {} for host {}.", m, host);
// If all is well, store the specific intervals.
hostSpecificMinInterval.put(host, m);
LOG.debug("Added custom min. interval {} for host {}.", m, host);

hostSpecificMaxInterval.put(host, M);
LOG.debug("Added custom max. interval {} for host {}.", M, host);
hostSpecificMaxInterval.put(host, M);
LOG.debug("Added custom max. interval {} for host {}.", M, host);

}
}
}

Expand Down
17 changes: 8 additions & 9 deletions src/java/org/apache/nutch/crawl/TextProfileSignature.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,17 +195,16 @@ public static void main(String[] args) throws Exception {
HashMap<String, byte[]> res = new HashMap<>();
File[] files = new File(args[0]).listFiles();
for (int i = 0; i < files.length; i++) {
FileInputStream fis = new FileInputStream(files[i]);
BufferedReader br = new BufferedReader(
new InputStreamReader(fis, StandardCharsets.UTF_8));
StringBuffer text = new StringBuffer();
String line = null;
while ((line = br.readLine()) != null) {
if (text.length() > 0)
text.append("\n");
text.append(line);
try (BufferedReader br = new BufferedReader(new InputStreamReader(
new FileInputStream(files[i]), StandardCharsets.UTF_8))) {
String line = null;
while ((line = br.readLine()) != null) {
if (text.length() > 0)
text.append("\n");
text.append(line);
}
}
br.close();
byte[] signature = sig.calculate(null, new ParseImpl(text.toString(),
null));
res.put(files[i].toString(), signature);
Expand Down
48 changes: 24 additions & 24 deletions src/java/org/apache/nutch/protocol/RobotRulesParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -398,30 +398,30 @@ public int run(String[] args) {
System.out.println("Testing robots.txt for agent names: "
+ (agentNames.isEmpty() ? "* (any other agent)" : agentNames));

LineNumberReader testsIn = new LineNumberReader(
new FileReader(urlFile, StandardCharsets.UTF_8));
String testPath;
testPath = testsIn.readLine();
while (testPath != null) {
testPath = testPath.trim();
try {
// testPath can be just a path or a complete URL
URL url = new URL(testPath);
String status;
if (isAllowListed(url)) {
status = "allowlisted";
} else if (rules.isAllowed(testPath)) {
status = "allowed";
} else {
status = "not allowed";
try (LineNumberReader testsIn = new LineNumberReader(
new FileReader(urlFile, StandardCharsets.UTF_8))) {
String testPath;
testPath = testsIn.readLine();
while (testPath != null) {
testPath = testPath.trim();
try {
// testPath can be just a path or a complete URL
URL url = new URL(testPath);
String status;
if (isAllowListed(url)) {
status = "allowlisted";
} else if (rules.isAllowed(testPath)) {
status = "allowed";
} else {
status = "not allowed";
}
System.out.println(status + ":\t" + testPath);
} catch (MalformedURLException e) {
LOG.warn("Not a valid URL: {}", testPath);
}
System.out.println(status + ":\t" + testPath);
} catch (MalformedURLException e) {
LOG.warn("Not a valid URL: {}", testPath);
testPath = testsIn.readLine();
}
testPath = testsIn.readLine();
}
testsIn.close();
} catch (IOException e) {
LOG.error("Failed to run:", e);
return -1;
Expand Down Expand Up @@ -476,9 +476,9 @@ public BaseRobotRules getRobotRulesSet(Protocol protocol, URL url,
try {
int contentLength = url.openConnection().getContentLength();
byte[] robotsBytes = new byte[contentLength];
InputStream openStream = url.openStream();
openStream.read(robotsBytes);
openStream.close();
try (InputStream openStream = url.openStream()) {
openStream.read(robotsBytes);
}
rules = robotParser.parseContent(url.toString(), robotsBytes,
"text/plain", agentNames);
} catch (IOException e) {
Expand Down
33 changes: 17 additions & 16 deletions src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,25 +141,26 @@ private int runCounter(FileSystem fs, Path webGraphDb) throws IOException,
}
Path numLinksFile = numLinksFiles[0].getPath();
LOG.info("Reading numlinks temp file {}", numLinksFile);
FSDataInputStream readLinks = fs.open(numLinksFile);
CompressionCodecFactory cf = new CompressionCodecFactory(conf);
CompressionCodec codec = cf.getCodec(numLinksFiles[0].getPath());
InputStream streamLinks;
if (codec == null) {
LOG.debug("No compression codec found for {}, trying uncompressed",
numLinksFile);
streamLinks = readLinks;
} else {
LOG.info("Compression codec of numlinks temp file: {}",
codec.getDefaultExtension());
readLinks.seek(0);
streamLinks = codec.createInputStream(readLinks);
String numLinksLine;
try (FSDataInputStream readLinks = fs.open(numLinksFile)) {
InputStream streamLinks;
if (codec == null) {
LOG.debug("No compression codec found for {}, trying uncompressed",
numLinksFile);
streamLinks = readLinks;
} else {
LOG.info("Compression codec of numlinks temp file: {}",
codec.getDefaultExtension());
readLinks.seek(0);
streamLinks = codec.createInputStream(readLinks);
}
try (BufferedReader buffer = new BufferedReader(
new InputStreamReader(streamLinks, StandardCharsets.UTF_8))) {
numLinksLine = buffer.readLine();
}
}
BufferedReader buffer = new BufferedReader(
new InputStreamReader(streamLinks, StandardCharsets.UTF_8));

String numLinksLine = buffer.readLine();
readLinks.close();

// check if there are links to process, if none, webgraph might be empty
if (numLinksLine == null || numLinksLine.length() == 0) {
Expand Down
Loading
Loading