diff --git a/htroot/IndexControlURLs_p.java b/htroot/IndexControlURLs_p.java index 6b3319f1ef..1f15428376 100644 --- a/htroot/IndexControlURLs_p.java +++ b/htroot/IndexControlURLs_p.java @@ -323,7 +323,7 @@ private static serverObjects genUrlProfile(final Segment segment, final URIMetad } final URIMetadataRow.Components metadata = entry.metadata(); final URIMetadataRow le = ((entry.referrerHash() == null) || (entry.referrerHash().length() != Word.commonHashLength)) ? null : segment.urlMetadata().load(entry.referrerHash(), null, 0); - if (metadata.url() == null) { + if (metadata == null || metadata.url() == null) { prop.put("genUrlProfile", "1"); prop.put("genUrlProfile_urlhash", urlhash); return prop; diff --git a/htroot/yacy/transferURL.java b/htroot/yacy/transferURL.java index 155bf76b3d..77be8bfca3 100644 --- a/htroot/yacy/transferURL.java +++ b/htroot/yacy/transferURL.java @@ -109,7 +109,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje // check if entry is well-formed final URIMetadataRow.Components metadata = lEntry.metadata(); - if (metadata.url() == null) { + if (metadata == null || metadata.url() == null) { yacyCore.log.logWarning("transferURL: received invalid URL from peer " + otherPeerName + "\n\tURL Property: " + urls); blocked++; continue; diff --git a/source/de/anomic/search/DocumentIndex.java b/source/de/anomic/search/DocumentIndex.java index 8f4b1b1adc..09a0079adc 100644 --- a/source/de/anomic/search/DocumentIndex.java +++ b/source/de/anomic/search/DocumentIndex.java @@ -40,6 +40,7 @@ import net.yacy.document.ParserException; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; +import net.yacy.kelondro.data.meta.URIMetadataRow.Components; import net.yacy.kelondro.logging.Log; @@ -179,8 +180,11 @@ public void addConcurrent(File start) { public ArrayList find(String querystring, int pos, int count) { ArrayList result = findMetadata(querystring, this); ArrayList files = new ArrayList(); + Components metadata; for (URIMetadataRow row : result) { - files.add(row.metadata().url().getLocalFile()); + metadata = row.metadata(); + if (metadata == null) continue; + files.add(metadata.url().getLocalFile()); count--; if (count == 0) break; } diff --git a/source/de/anomic/search/RankingProcess.java b/source/de/anomic/search/RankingProcess.java index abb77c76ec..00793af870 100644 --- a/source/de/anomic/search/RankingProcess.java +++ b/source/de/anomic/search/RankingProcess.java @@ -44,6 +44,7 @@ import net.yacy.document.Condenser; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; +import net.yacy.kelondro.data.meta.URIMetadataRow.Components; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.data.word.WordReferenceVars; @@ -393,7 +394,7 @@ public URIMetadataRow takeURL(final boolean skipDoubleDom, final int timeout) { final URIMetadataRow.Components metadata = page.metadata(); // check url constraints - if (metadata.url() == null) { + if (metadata == null || metadata.url() == null) { continue; // rare case where the url is corrupted } @@ -583,10 +584,13 @@ public ArrayList getHostNavigator(int count) { URIMetadataRow mr; DigestURI url; String hostname; + Components metadata; loop: for (int i = 0; i < rc; i++) { mr = this.query.getSegment().urlMetadata().load(hsa[i].hashsample, null, 0); if (mr == null) continue; - url = mr.metadata().url(); + metadata = mr.metadata(); + if (metadata == null) continue; + url = metadata.url(); if (url == null) continue; hostname = url.getHost(); if (hostname == null) continue; diff --git a/source/de/anomic/search/ResultFetcher.java b/source/de/anomic/search/ResultFetcher.java index 4ca9d19fc5..3996c73654 100644 --- a/source/de/anomic/search/ResultFetcher.java +++ b/source/de/anomic/search/ResultFetcher.java @@ -201,6 +201,7 @@ protected ResultEntry fetchSnippet(final URIMetadataRow page, final int snippetM long startTime = System.currentTimeMillis(); final URIMetadataRow.Components metadata = page.metadata(); + if (metadata == null) return null; final long dbRetrievalTime = System.currentTimeMillis() - startTime; if (snippetMode == 0) { diff --git a/source/de/anomic/search/Segment.java b/source/de/anomic/search/Segment.java index c3f1e28139..b0ad82a6f1 100644 --- a/source/de/anomic/search/Segment.java +++ b/source/de/anomic/search/Segment.java @@ -371,7 +371,7 @@ public int removeAllUrlReferences(final String urlhash, LoaderDispatcher loader, final URIMetadataRow entry = urlMetadata().load(urlhash, null, 0); if (entry == null) return 0; final URIMetadataRow.Components metadata = entry.metadata(); - if (metadata.url() == null) return 0; + if (metadata == null || metadata.url() == null) return 0; InputStream resourceContent = null; try { diff --git a/source/de/anomic/search/Switchboard.java b/source/de/anomic/search/Switchboard.java index b539d15b73..fdc62d088c 100644 --- a/source/de/anomic/search/Switchboard.java +++ b/source/de/anomic/search/Switchboard.java @@ -73,6 +73,7 @@ import net.yacy.kelondro.blob.BEncodedHeapArray; import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; +import net.yacy.kelondro.data.meta.URIMetadataRow.Components; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.RowSpaceExceededException; import net.yacy.kelondro.logging.Log; @@ -997,8 +998,10 @@ public DigestURI getURL(final Segments.Process process, final String urlhash) { final DigestURI ne = crawlQueues.getURL(urlhash); if (ne != null) return ne; final URIMetadataRow le = indexSegments.urlMetadata(process).load(urlhash, null, 0); - if (le != null) return le.metadata().url(); - return null; + if (le == null) return null; + Components metadata = le.metadata(); + if (metadata == null) return null; + return metadata.url(); } public RankingProfile getRanking() { diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 8d44b4cf6d..632e912e05 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -556,6 +556,7 @@ public static String[] search( assert (urlEntry.hash().length() == 12) : "urlEntry.hash() = " + urlEntry.hash(); if (urlEntry.hash().length() != 12) continue; // bad url hash final URIMetadataRow.Components metadata = urlEntry.metadata(); + if (metadata == null) continue; if (blacklist.isListed(Blacklist.BLACKLIST_SEARCH, metadata.url())) { yacyCore.log.logInfo("remote search (client): filtered blacklisted url " + metadata.url() + " from peer " + target.getName()); continue; // block with backlist diff --git a/source/net/yacy/kelondro/blob/MapHeap.java b/source/net/yacy/kelondro/blob/MapHeap.java index 6091e9b5ac..661ad4396d 100644 --- a/source/net/yacy/kelondro/blob/MapHeap.java +++ b/source/net/yacy/kelondro/blob/MapHeap.java @@ -218,7 +218,7 @@ public Map get(final String key) throws IOException { } private String normalizeKey(String key) { - if (blob == null) return key; + if (blob == null || key == null) return key; if (key.length() > blob.keylength()) key = key.substring(0, blob.keylength()); while (key.length() < blob.keylength()) key += fillchar; return key; diff --git a/source/net/yacy/kelondro/data/meta/URIMetadataRow.java b/source/net/yacy/kelondro/data/meta/URIMetadataRow.java index 0a07e50001..d9c3d6523e 100644 --- a/source/net/yacy/kelondro/data/meta/URIMetadataRow.java +++ b/source/net/yacy/kelondro/data/meta/URIMetadataRow.java @@ -307,6 +307,7 @@ private StringBuilder corePropList() { // generate a parseable string; this is a simple property-list final Components metadata = this.metadata(); final StringBuilder s = new StringBuilder(300); + if (metadata == null) return null; //System.out.println("author=" + comp.author()); try { s.append("hash=").append(hash());