diff --git a/.gitignore b/.gitignore index f04e658173..277ddceff5 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ RELEASE/ lib/yacy-cora.jar /DATA.bkp /DATA.1 +/gen diff --git a/htroot/BlacklistTest_p.java b/htroot/BlacklistTest_p.java index 3fe6a88b74..429a7de401 100644 --- a/htroot/BlacklistTest_p.java +++ b/htroot/BlacklistTest_p.java @@ -31,8 +31,8 @@ import java.net.MalformedURLException; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; @@ -55,9 +55,9 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea !urlstring.startsWith("ftp://") && !urlstring.startsWith("smb://") && !urlstring.startsWith("file://")) urlstring = "http://" + urlstring; - DigestURI testurl = null; + DigestURL testurl = null; try { - testurl = new DigestURI(urlstring); + testurl = new DigestURL(urlstring); } catch (final MalformedURLException e) { testurl = null; } diff --git a/htroot/Blacklist_p.java b/htroot/Blacklist_p.java index ed86da3a01..163225971e 100644 --- a/htroot/Blacklist_p.java +++ b/htroot/Blacklist_p.java @@ -35,12 +35,12 @@ import java.util.Arrays; import java.util.List; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.ListManager; import net.yacy.data.WorkTables; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.Blacklist; import net.yacy.repository.Blacklist.BlacklistType; @@ -81,9 +81,9 @@ public static serverObjects respond(final RequestHeader header, final serverObje !urlstring.startsWith("file://")) { urlstring = "http://"+urlstring; } - DigestURI testurl; + DigestURL testurl; try { - testurl = new DigestURI(urlstring); + testurl = new DigestURL(urlstring); } catch (final MalformedURLException e) { testurl = null; } diff --git a/htroot/Blog.java b/htroot/Blog.java index 41d495a57a..9ddbef9eed 100644 --- a/htroot/Blog.java +++ b/htroot/Blog.java @@ -38,7 +38,7 @@ import java.util.Locale; import java.util.Map; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; diff --git a/htroot/BlogComments.java b/htroot/BlogComments.java index e8650df5bb..26406cffc4 100644 --- a/htroot/BlogComments.java +++ b/htroot/BlogComments.java @@ -35,7 +35,7 @@ import java.util.Date; import java.util.Iterator; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; diff --git a/htroot/Bookmarks.java b/htroot/Bookmarks.java index e9cb45f06a..1c5b9595c9 100644 --- a/htroot/Bookmarks.java +++ b/htroot/Bookmarks.java @@ -39,7 +39,8 @@ import java.util.Set; import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.HeaderFramework; @@ -53,7 +54,6 @@ import net.yacy.data.BookmarksDB.Tag; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.peers.NewsPool; import net.yacy.search.Switchboard; @@ -247,7 +247,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje try { final File file = new File(post.get("htmlfile")); - BookmarkHelper.importFromBookmarks(sb.bookmarksDB, new DigestURI(file), post.get("htmlfile$file"), tags, isPublic); + BookmarkHelper.importFromBookmarks(sb.bookmarksDB, new DigestURL(file), post.get("htmlfile$file"), tags, isPublic); } catch (final MalformedURLException e) {} ConcurrentLog.info("BOOKMARKS", "success!!"); diff --git a/htroot/CacheResource_p.java b/htroot/CacheResource_p.java index 3cb2ec8355..caa95d1bab 100644 --- a/htroot/CacheResource_p.java +++ b/htroot/CacheResource_p.java @@ -24,13 +24,13 @@ import java.net.MalformedURLException; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.ResponseHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.data.Cache; import net.yacy.document.ImageParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; import net.yacy.server.servletProperties; @@ -44,9 +44,9 @@ public static Object respond(final RequestHeader header, final serverObjects pos if (post == null) return prop; final String u = post.get("url", ""); - DigestURI url; + DigestURL url; try { - url = new DigestURI(u); + url = new DigestURL(u); } catch (final MalformedURLException e) { ConcurrentLog.logException(e); return prop; diff --git a/htroot/Collage.java b/htroot/Collage.java index b32fc1b9c0..1ac10e9aa7 100644 --- a/htroot/Collage.java +++ b/htroot/Collage.java @@ -24,7 +24,7 @@ import java.util.Random; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.RequestHeader; import net.yacy.crawler.data.ResultImages; @@ -89,8 +89,8 @@ public static serverObjects respond(final RequestHeader header, final serverObje final int yOffset = embed ? 0 : 70; for (int i = 0; i < fifoSize; i++) { - final MultiProtocolURI baseURL = origins[i].baseURL; - final MultiProtocolURI imageURL = origins[i].imageEntry.url(); + final MultiProtocolURL baseURL = origins[i].baseURL; + final MultiProtocolURL imageURL = origins[i].imageEntry.url(); // check if this loads a page from localhost, which must be prevented to protect the server // against attacks to the administration interface when localhost access is granted diff --git a/htroot/ConfigAppearance_p.java b/htroot/ConfigAppearance_p.java index 83b37ee487..3791f9e8bf 100644 --- a/htroot/ConfigAppearance_p.java +++ b/htroot/ConfigAppearance_p.java @@ -39,9 +39,9 @@ import java.util.List; import java.util.Map; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; @@ -101,7 +101,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea final Iterator it; try { - final DigestURI u = new DigestURI(url); + final DigestURL u = new DigestURL(url); it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent)); } catch (final IOException e) { prop.put("status", "1");// unable to get URL diff --git a/htroot/ConfigHeuristics_p.java b/htroot/ConfigHeuristics_p.java index 8ea27b15c1..0c7fe70626 100644 --- a/htroot/ConfigHeuristics_p.java +++ b/htroot/ConfigHeuristics_p.java @@ -239,7 +239,7 @@ private static void writeopensearchcfg(final Switchboard sb, final serverObjects // re-read config (and create/update work table) if (sb.getConfigBool(SwitchboardConstants.HEURISTIC_OPENSEARCH, true)) { - OpenSearchConnector os = new OpenSearchConnector(sb, true); + new OpenSearchConnector(sb, true); } } } diff --git a/htroot/ConfigLanguage_p.java b/htroot/ConfigLanguage_p.java index f1103e7949..b307d2c9a3 100644 --- a/htroot/ConfigLanguage_p.java +++ b/htroot/ConfigLanguage_p.java @@ -39,11 +39,11 @@ import java.util.List; import java.util.Map; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; import net.yacy.data.Translator; import net.yacy.data.WorkTables; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; @@ -100,7 +100,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea final String url = post.get("url"); Iterator it; try { - final DigestURI u = new DigestURI(url); + final DigestURL u = new DigestURL(url); it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent)); } catch(final IOException e) { prop.put("status", "1");//unable to get url diff --git a/htroot/ConfigNetwork_p.java b/htroot/ConfigNetwork_p.java index 731c19457d..75ae2f39e9 100644 --- a/htroot/ConfigNetwork_p.java +++ b/htroot/ConfigNetwork_p.java @@ -30,7 +30,7 @@ import java.io.IOException; import java.util.Set; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.protocol.RequestHeader; import net.yacy.data.WorkTables; import net.yacy.kelondro.util.FileUtils; diff --git a/htroot/ConfigPortal.java b/htroot/ConfigPortal.java index 561ada482b..9c83ec03e0 100644 --- a/htroot/ConfigPortal.java +++ b/htroot/ConfigPortal.java @@ -30,10 +30,11 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.Properties; + +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.WorkTables; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverObjects; @@ -98,7 +99,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje String excludehosts = post.get("search.excludehosts", ""); sb.setConfig("search.excludehosts", excludehosts); - sb.setConfig("search.excludehosth", DigestURI.hosthashes(excludehosts)); + sb.setConfig("search.excludehosth", DigestURL.hosthashes(excludehosts)); } if (post.containsKey("searchpage_default")) { // load defaults from defaults/yacy.init file diff --git a/htroot/ConfigUpdate_p.java b/htroot/ConfigUpdate_p.java index 71bcdd7d5f..c3f6a58d0e 100644 --- a/htroot/ConfigUpdate_p.java +++ b/htroot/ConfigUpdate_p.java @@ -32,9 +32,9 @@ import java.util.Set; import java.util.TreeSet; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.OS; import net.yacy.peers.operation.yacyBuildProperties; @@ -86,7 +86,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea final String release = post.get("releasedownload", ""); if (!release.isEmpty()) { try { - yacyRelease versionToDownload = new yacyRelease(new DigestURI(release)); + yacyRelease versionToDownload = new yacyRelease(new DigestURL(release)); // replace this version with version which contains public key final yacyRelease.DevAndMainVersions allReleases = yacyRelease.allReleases(false, false); diff --git a/htroot/CrawlCheck_p.java b/htroot/CrawlCheck_p.java index c6c660dd23..09f342b94c 100644 --- a/htroot/CrawlCheck_p.java +++ b/htroot/CrawlCheck_p.java @@ -24,6 +24,7 @@ import java.util.Set; import java.util.regex.Pattern; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; @@ -31,7 +32,6 @@ import net.yacy.crawler.retrieval.Request; import net.yacy.crawler.retrieval.Response; import net.yacy.crawler.robots.RobotsTxtEntry; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; @@ -49,7 +49,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (post.containsKey("crawlcheck")) { // get the list of rootURls for this crawl start - Set rootURLs = new HashSet(); + Set rootURLs = new HashSet(); String crawlingStart0 = post.get("crawlingURLs","").trim(); String[] rootURLs0 = crawlingStart0.indexOf('\n') > 0 || crawlingStart0.indexOf('\r') > 0 ? crawlingStart0.split("[\\r\\n]+") : crawlingStart0.split(Pattern.quote("|")); for (String crawlingStart: rootURLs0) { @@ -61,7 +61,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (crawlingStart.startsWith("ftp")) crawlingStart = "ftp://" + crawlingStart; } try { - DigestURI crawlingStartURL = new DigestURI(crawlingStart); + DigestURL crawlingStartURL = new DigestURL(crawlingStart); rootURLs.add(crawlingStartURL); } catch (final MalformedURLException e) { ConcurrentLog.logException(e); @@ -78,7 +78,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea // and analyze the urls to make the table rows StringBuilder s = new StringBuilder(300); int row = 0; - for (DigestURI u: rootURLs) { + for (DigestURL u: rootURLs) { s.append(u.toNormalform(true)).append('\n'); prop.put("table_list_" + row + "_url", u.toNormalform(true)); diff --git a/htroot/CrawlResults.java b/htroot/CrawlResults.java index c8fb7f2580..0ee465dbfd 100644 --- a/htroot/CrawlResults.java +++ b/htroot/CrawlResults.java @@ -31,8 +31,8 @@ import java.util.Locale; import java.util.Map; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.data.ResultURLs; diff --git a/htroot/CrawlStartScanner_p.java b/htroot/CrawlStartScanner_p.java index f3e2942c2d..400824c194 100644 --- a/htroot/CrawlStartScanner_p.java +++ b/htroot/CrawlStartScanner_p.java @@ -31,6 +31,7 @@ import java.util.TreeMap; import java.util.regex.Pattern; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.solr.connector.AbstractSolrConnector; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.Domains; @@ -40,7 +41,6 @@ import net.yacy.cora.sorting.ReversibleScoreMap; import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.WorkTables; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.search.query.SearchEventCache; @@ -193,10 +193,10 @@ public static serverObjects respond( if ( post.containsKey("crawl") ) { // make a pk/url mapping final Iterator> se = Scanner.scancacheEntries(); - final Map pkmap = new TreeMap(Base64Order.enhancedCoder); + final Map pkmap = new TreeMap(Base64Order.enhancedCoder); while (se.hasNext()) { final Scanner.Service u = se.next().getKey(); - DigestURI uu; + DigestURL uu; try { uu = u.url(); pkmap.put(uu.hash(), uu); @@ -208,7 +208,7 @@ public static serverObjects respond( for ( final Map.Entry entry : post.entrySet() ) { if ( entry.getValue().startsWith("mark_") ) { final byte[] pk = entry.getValue().substring(5).getBytes(); - final DigestURI url = pkmap.get(pk); + final DigestURL url = pkmap.get(pk); if ( url != null ) { String path = "/Crawler_p.html?createBookmark=off&xsstopw=off&crawlingDomMaxPages=10000&intention=&range=domain&indexMedia=on&recrawl=nodoubles&xdstopw=off&storeHTCache=on&sitemapURL=&repeat_time=7&crawlingQ=on&cachePolicy=iffresh&indexText=on&crawlingMode=url&mustnotmatch=&crawlingDomFilterDepth=1&crawlingDomFilterCheck=off&crawlingstart=Start%20New%20Crawl&xpstopw=off&repeat_unit=seldays&crawlingDepth=99&directDocByURL=off"; path += "&crawlingURL=" + url.toNormalform(true); @@ -244,7 +244,7 @@ public static serverObjects respond( final Map apiCommentCache = WorkTables.commentCache(sb); String urlString; - DigestURI u; + DigestURL u; try { final Iterator> se = Scanner.scancacheEntries(); Map.Entry host; diff --git a/htroot/Crawler_p.java b/htroot/Crawler_p.java index 0bcde88ad6..e972b5fa0d 100644 --- a/htroot/Crawler_p.java +++ b/htroot/Crawler_p.java @@ -30,12 +30,13 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.Set; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.AnchorURL; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; @@ -50,7 +51,6 @@ import net.yacy.document.Document; import net.yacy.document.parser.html.ContentScraper; import net.yacy.document.parser.html.TransformerWriter; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.index.RowHandleSet; import net.yacy.kelondro.util.FileUtils; import net.yacy.peers.NewsPool; @@ -175,7 +175,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje String crawlingStart0 = post.get("crawlingURL","").trim(); // the crawljob start url String[] rootURLs0 = crawlingStart0.indexOf('\n') > 0 || crawlingStart0.indexOf('\r') > 0 ? crawlingStart0.split("[\\r\\n]+") : crawlingStart0.split(Pattern.quote("|")); - Set rootURLs = new HashSet(); + Set rootURLs = new HashSet(); String crawlName = ""; if (crawlingFile == null) for (String crawlingStart: rootURLs0) { if (crawlingStart == null || crawlingStart.length() == 0) continue; @@ -185,7 +185,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje if (crawlingStart.startsWith("ftp")) crawlingStart = "ftp://" + crawlingStart; else crawlingStart = "http://" + crawlingStart; } try { - DigestURI crawlingStartURL = new DigestURI(crawlingStart); + DigestURL crawlingStartURL = new DigestURL(crawlingStart); rootURLs.add(crawlingStartURL); crawlName += ((crawlingStartURL.getHost() == null) ? crawlingStartURL.toNormalform(true) : crawlingStartURL.getHost()) + ','; if (crawlingStartURL != null && (crawlingStartURL.isFile() || crawlingStartURL.isSMB())) storeHTCache = false; @@ -288,14 +288,14 @@ public static serverObjects respond(final RequestHeader header, final serverObje if ("sitelist".equals(crawlingMode)) { newcrawlingMustNotMatch = CrawlProfile.MATCH_NEVER_STRING; - Set newRootURLs = new HashSet(); - for (DigestURI sitelistURL: rootURLs) { + Set newRootURLs = new HashSet(); + for (DigestURL sitelistURL: rootURLs) { // download document Document scraper; try { scraper = sb.loader.loadDocument(sitelistURL, CacheStrategy.IFFRESH, BlacklistType.CRAWLER, agent); // get links and generate filter - for (DigestURI u: scraper.getAnchors().keySet()) { + for (DigestURL u: scraper.getAnchors()) { newRootURLs.add(u); } } catch (final IOException e) { @@ -313,14 +313,14 @@ public static serverObjects respond(final RequestHeader header, final serverObje if (fullDomain) { siteFilter = CrawlProfile.siteFilter(rootURLs); if (deleteold) { - for (DigestURI u: rootURLs) { + for (DigestURL u: rootURLs) { sb.index.fulltext().deleteDomainHashpart(u.hosthash(), deleteageDate); } } } else if (subPath) { siteFilter = CrawlProfile.subpathFilter(rootURLs); if (deleteold) { - for (DigestURI u: rootURLs) { + for (DigestURL u: rootURLs) { String basepath = u.toNormalform(true); if (!basepath.endsWith("/")) {int p = basepath.lastIndexOf("/"); if (p > 0) basepath = basepath.substring(0, p + 1);} int count = sb.index.fulltext().remove(basepath, deleteageDate); @@ -339,7 +339,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje // check if the crawl filter works correctly try { Pattern mmp = Pattern.compile(newcrawlingMustMatch); - for (DigestURI u: rootURLs) { + for (DigestURL u: rootURLs) { assert mmp.matcher(u.toNormalform(true)).matches() : "pattern " + mmp.toString() + " does not match url " + u.toNormalform(true); } } catch (final PatternSyntaxException e) { @@ -389,7 +389,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje // delete all error urls for that domain List hosthashes = new ArrayList(); - for (DigestURI u: rootURLs) { + for (DigestURL u: rootURLs) { hosthashes.add(ASCII.getBytes(u.hosthash())); } sb.crawlQueues.errorURL.removeHosts(hosthashes, false); @@ -411,8 +411,8 @@ public static serverObjects respond(final RequestHeader header, final serverObje // stack requests sb.crawler.putActive(handle, profile); - final Set successurls = new HashSet(); - final Map failurls = new HashMap(); + final Set successurls = new HashSet(); + final Map failurls = new HashMap(); sb.stackURLs(rootURLs, profile, successurls, failurls); if (failurls.size() == 0) { @@ -439,7 +439,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje } } else { StringBuilder fr = new StringBuilder(); - for (Map.Entry failure: failurls.entrySet()) { + for (Map.Entry failure: failurls.entrySet()) { sb.crawlQueues.errorURL.push( new Request( sb.peers.mySeed().hash.getBytes(), @@ -470,7 +470,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje } else if ("sitemap".equals(crawlingMode)) { final String sitemapURLStr = post.get("sitemapURL",""); try { - final DigestURI sitemapURL = new DigestURI(sitemapURLStr); + final DigestURL sitemapURL = new DigestURL(sitemapURLStr); sb.crawler.putActive(handle, profile); final SitemapImporter importer = new SitemapImporter(sb, sitemapURL, profile); importer.start(); @@ -488,7 +488,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje try { // check if the crawl filter works correctly Pattern.compile(newcrawlingMustMatch); - final ContentScraper scraper = new ContentScraper(new DigestURI(crawlingFile), 10000000); + final ContentScraper scraper = new ContentScraper(new DigestURL(crawlingFile), 10000000); final Writer writer = new TransformerWriter(null, null, scraper, null, false); if (crawlingFile != null && crawlingFile.exists()) { FileUtils.copy(new FileInputStream(crawlingFile), writer); @@ -498,12 +498,12 @@ public static serverObjects respond(final RequestHeader header, final serverObje writer.close(); // get links and generate filter - final Map hyperlinks = scraper.getAnchors(); + final List hyperlinks = scraper.getAnchors(); if (newcrawlingdepth > 0) { if (fullDomain) { - newcrawlingMustMatch = CrawlProfile.siteFilter(hyperlinks.keySet()); + newcrawlingMustMatch = CrawlProfile.siteFilter(hyperlinks); } else if (subPath) { - newcrawlingMustMatch = CrawlProfile.subpathFilter(hyperlinks.keySet()); + newcrawlingMustMatch = CrawlProfile.subpathFilter(hyperlinks); } } diff --git a/htroot/DictionaryLoader_p.java b/htroot/DictionaryLoader_p.java index 2aab571131..03c79d4a5a 100644 --- a/htroot/DictionaryLoader_p.java +++ b/htroot/DictionaryLoader_p.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.net.MalformedURLException; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.geo.GeonamesLocation; import net.yacy.cora.geo.OpenGeoDBLocation; @@ -29,7 +30,6 @@ import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.retrieval.Response; import net.yacy.document.LibraryProvider; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; @@ -66,7 +66,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (post.containsKey("geon0Load")) { // load from the net try { - final Response response = sb.loader.load(sb.loader.request(new DigestURI(LibraryProvider.Dictionary.GEON0.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent); + final Response response = sb.loader.load(sb.loader.request(new DigestURL(LibraryProvider.Dictionary.GEON0.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent); final byte[] b = response.getContent(); FileUtils.copy(b, LibraryProvider.Dictionary.GEON0.file()); LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEON0.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON0.file(), null, -1)); @@ -108,7 +108,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (post.containsKey("geon1Load")) { // load from the net try { - final Response response = sb.loader.load(sb.loader.request(new DigestURI(LibraryProvider.Dictionary.GEON1.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent); + final Response response = sb.loader.load(sb.loader.request(new DigestURL(LibraryProvider.Dictionary.GEON1.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent); final byte[] b = response.getContent(); FileUtils.copy(b, LibraryProvider.Dictionary.GEON1.file()); LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEON1.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON1.file(), null, -1)); @@ -150,7 +150,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (post.containsKey("geon2Load")) { // load from the net try { - final Response response = sb.loader.load(sb.loader.request(new DigestURI(LibraryProvider.Dictionary.GEON2.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent); + final Response response = sb.loader.load(sb.loader.request(new DigestURL(LibraryProvider.Dictionary.GEON2.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent); final byte[] b = response.getContent(); FileUtils.copy(b, LibraryProvider.Dictionary.GEON2.file()); LibraryProvider.geoLoc.activateLocation(LibraryProvider.Dictionary.GEON2.nickname, new GeonamesLocation(LibraryProvider.Dictionary.GEON2.file(), null, 100000)); @@ -192,7 +192,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (post.containsKey("geo1Load")) { // load from the net try { - final Response response = sb.loader.load(sb.loader.request(new DigestURI(LibraryProvider.Dictionary.GEODB1.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent); + final Response response = sb.loader.load(sb.loader.request(new DigestURL(LibraryProvider.Dictionary.GEODB1.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent); final byte[] b = response.getContent(); FileUtils.copy(b, LibraryProvider.Dictionary.GEODB1.file()); LibraryProvider.geoLoc.deactivateLocalization(LibraryProvider.Dictionary.GEODB1.nickname); @@ -235,7 +235,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (post.containsKey("drw0Load")) { // load from the net try { - final Response response = sb.loader.load(sb.loader.request(new DigestURI(LibraryProvider.Dictionary.DRW0.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent); + final Response response = sb.loader.load(sb.loader.request(new DigestURL(LibraryProvider.Dictionary.DRW0.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent); final byte[] b = response.getContent(); FileUtils.copy(b, LibraryProvider.Dictionary.DRW0.file()); LibraryProvider.activateDeReWo(); @@ -279,7 +279,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (post.containsKey("pnd0Load")) { // load from the net try { - final Response response = sb.loader.load(sb.loader.request(new DigestURI(LibraryProvider.Dictionary.PND0.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent); + final Response response = sb.loader.load(sb.loader.request(new DigestURL(LibraryProvider.Dictionary.PND0.url), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, ClientIdentification.yacyInternetCrawlerAgent); final byte[] b = response.getContent(); FileUtils.copy(b, LibraryProvider.Dictionary.PND0.file()); LibraryProvider.activatePND(); diff --git a/htroot/HostBrowser.java b/htroot/HostBrowser.java index dd2f347cda..4b4df0c223 100644 --- a/htroot/HostBrowser.java +++ b/htroot/HostBrowser.java @@ -33,9 +33,10 @@ import org.apache.solr.common.SolrDocument; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.solr.FailType; import net.yacy.cora.federate.solr.connector.AbstractSolrConnector; import net.yacy.cora.protocol.RequestHeader; @@ -46,7 +47,6 @@ import net.yacy.crawler.HarvestProcess; import net.yacy.crawler.data.NoticedURL.StackType; import net.yacy.crawler.retrieval.Request; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.peers.graphics.WebStructureGraph.StructureEntry; import net.yacy.search.Switchboard; @@ -115,8 +115,8 @@ public static serverObjects respond(final RequestHeader header, final serverObje prop.putHTML("path", path); prop.put("delete", admin && path.length() > 0 ? 1 : 0); - DigestURI pathURI = null; - try {pathURI = new DigestURI(path);} catch (final MalformedURLException e) {} + DigestURL pathURI = null; + try {pathURI = new DigestURL(path);} catch (final MalformedURLException e) {} String load = post.get("load", ""); boolean wait = false; @@ -127,10 +127,10 @@ public static serverObjects respond(final RequestHeader header, final serverObje } if (load.length() > 0 && loadRight) { // stack URL - DigestURI url; + DigestURL url; if (sb.crawlStacker.size() > 2) wait = false; try { - url = new DigestURI(load); + url = new DigestURL(load); String reasonString = sb.crawlStacker.stackCrawl(new Request( sb.peers.mySeed().hash.getBytes(), url, null, load, new Date(), @@ -244,7 +244,7 @@ url, null, load, new Date(), } try { // generate file list from path - DigestURI uri = new DigestURI(path); + DigestURL uri = new DigestURL(path); String host = uri.getHost(); prop.putHTML("outbound_host", host); if (admin) prop.putHTML("outbound_admin_host", host); //used for WebStructurePicture_p link @@ -322,7 +322,7 @@ url, null, load, new Date(), while (links.hasNext()) { u = links.next(); try { - MultiProtocolURI mu = new MultiProtocolURI(u); + MultiProtocolURL mu = new MultiProtocolURL(u); if (mu.getHost() != null) { ReversibleScoreMap lks = outboundHosts.get(mu.getHost()); if (lks == null) { @@ -422,7 +422,7 @@ url, null, load, new Date(), prop.put("files_list_" + c + "_type", 0); prop.put("files_list_" + c + "_type_url", entry.getKey()); StoreType type = (StoreType) entry.getValue(); - try {uri = new DigestURI(entry.getKey());} catch (final MalformedURLException e) {uri = null;} + try {uri = new DigestURL(entry.getKey());} catch (final MalformedURLException e) {uri = null;} HarvestProcess process = uri == null ? null : sb.crawlQueues.exists(uri.hash()); boolean loading = load.equals(entry.getKey()) || (process != null && process != HarvestProcess.ERRORS); boolean error = process == HarvestProcess.ERRORS || type == StoreType.EXCLUDED || type == StoreType.FAILED; @@ -541,12 +541,12 @@ public InfoCacheEntry(final Fulltext fulltext, final ReferenceReportCache rrCach // get all urls from the index and store them here for (String id: internalIDs) { if (id.equals(urlhash)) continue; // no self-references - DigestURI u = fulltext.getURL(ASCII.getBytes(id)); + DigestURL u = fulltext.getURL(ASCII.getBytes(id)); if (u != null) references_internal_urls.add(u.toNormalform(true)); } for (String id: externalIDs) { if (id.equals(urlhash)) continue; // no self-references - DigestURI u = fulltext.getURL(ASCII.getBytes(id)); + DigestURL u = fulltext.getURL(ASCII.getBytes(id)); if (u != null) references_external_urls.add(u.toNormalform(true)); } } catch (final IOException e) { diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index b04aa59178..b95d284d7f 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -31,8 +31,9 @@ import java.util.List; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; import net.yacy.cora.document.analysis.Classification.ContentDomain; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; @@ -42,7 +43,6 @@ import net.yacy.cora.util.SpaceExceededException; import net.yacy.data.ListManager; import net.yacy.document.Condenser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.word.Word; @@ -368,7 +368,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if ( post.containsKey("blacklisturls") ) { final String[] supportedBlacklistTypes = env.getConfig("BlackLists.types", "").split(","); - DigestURI url; + DigestURL url; for ( final byte[] b : urlb ) { try { urlHashes.put(b); @@ -395,7 +395,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea } if ( post.containsKey("blacklistdomains") ) { - DigestURI url; + DigestURL url; for ( final byte[] b : urlb ) { try { urlHashes.put(b); @@ -461,7 +461,7 @@ public static void genURLList( prop.put("genUrlList_flags", (flags == null) ? "" : flags.exportB64()); prop.put("genUrlList_lines", maxlines); int i = 0; - DigestURI url; + DigestURL url; URIMetadataNode entry; String us; long rn = -1; @@ -483,7 +483,7 @@ public static void genURLList( prop.put("genUrlList_urlList_" + i + "_urlExists_urlStringShort", (us.length() > 40) ? (us.substring(0, 20) + "
" + us.substring(20, 40) + "...") : ((us.length() > 30) ? (us.substring(0, 20) + "
" + us.substring(20)) : us)); prop.putNum("genUrlList_urlList_" + i + "_urlExists_ranking", (entry.ranking() - rn)); - prop.putNum("genUrlList_urlList_" + i + "_urlExists_domlength", DigestURI.domLengthEstimation(entry.hash())); + prop.putNum("genUrlList_urlList_" + i + "_urlExists_domlength", DigestURL.domLengthEstimation(entry.hash())); prop.putNum("genUrlList_urlList_" + i + "_urlExists_tf", 1000.0 * entry.word().termFrequency()); prop.putNum("genUrlList_urlList_" + i + "_urlExists_authority", (theSearch.getOrder() == null) ? -1 : theSearch.getOrder().authority(ASCII.String(entry.hash(), 6, 6))); prop.put("genUrlList_urlList_" + i + "_urlExists_date", GenericFormatter.SHORT_DAY_FORMATTER.format(new Date(entry.word().lastModified()))); @@ -627,7 +627,7 @@ public static SearchEvent genSearchresult( filter, false, null, - DigestURI.TLD_any_zone_filter, + DigestURL.TLD_any_zone_filter, "", false, sb.index, diff --git a/htroot/IndexControlURLs_p.java b/htroot/IndexControlURLs_p.java index 9f202d2708..1fc8c91d87 100644 --- a/htroot/IndexControlURLs_p.java +++ b/htroot/IndexControlURLs_p.java @@ -33,7 +33,8 @@ import java.util.Map; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.lod.JenaTripleStore; import net.yacy.cora.protocol.ClientIdentification; @@ -43,7 +44,6 @@ import net.yacy.crawler.data.Cache; import net.yacy.crawler.data.ResultURLs; import net.yacy.data.WorkTables; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.word.Word; import net.yacy.search.Switchboard; @@ -127,7 +127,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea String urlhash = post.get("urlhash", "").trim(); if (urlhash.isEmpty() && urlstring.length() > 0) { try { - urlhash = ASCII.String(new DigestURI(urlstring).hash()); + urlhash = ASCII.String(new DigestURL(urlstring).hash()); } catch (final MalformedURLException e) { } } @@ -184,7 +184,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea } if (post.containsKey("urlhashdelete")) { - final DigestURI url = segment.fulltext().getURL(ASCII.getBytes(urlhash)); + final DigestURL url = segment.fulltext().getURL(ASCII.getBytes(urlhash)); if (url == null) { prop.putHTML("result", "No Entry for URL hash " + urlhash + "; nothing deleted."); } else { @@ -197,7 +197,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (post.containsKey("urldelete")) { try { - urlhash = ASCII.String((new DigestURI(urlstring)).hash()); + urlhash = ASCII.String((new DigestURL(urlstring)).hash()); } catch (final MalformedURLException e) { urlhash = null; } @@ -211,7 +211,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (post.containsKey("urlstringsearch")) { try { - final DigestURI url = new DigestURI(urlstring); + final DigestURL url = new DigestURL(urlstring); urlhash = ASCII.String(url.hash()); prop.put("urlhash", urlhash); final URIMetadataNode entry = segment.fulltext().getMetadata(ASCII.getBytes(urlhash)); diff --git a/htroot/IndexCreateLoaderQueue_p.java b/htroot/IndexCreateLoaderQueue_p.java index 85b275da95..0d89f8319a 100644 --- a/htroot/IndexCreateLoaderQueue_p.java +++ b/htroot/IndexCreateLoaderQueue_p.java @@ -27,7 +27,7 @@ // javac -classpath .:../classes IndexCreate_p.java // if the shell's current path is HTROOT -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.protocol.RequestHeader; import net.yacy.crawler.retrieval.Request; import net.yacy.peers.Seed; diff --git a/htroot/IndexCreateParserErrors_p.java b/htroot/IndexCreateParserErrors_p.java index 3133e112c6..6a10f44de5 100644 --- a/htroot/IndexCreateParserErrors_p.java +++ b/htroot/IndexCreateParserErrors_p.java @@ -26,11 +26,11 @@ import java.util.ArrayList; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.RequestHeader; import net.yacy.crawler.CrawlStacker; import net.yacy.crawler.data.ZURL; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.peers.Seed; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; @@ -72,7 +72,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea prop.put("rejected_only-latest", "0"); } dark = true; - DigestURI url; + DigestURL url; byte[] initiatorHash, executorHash; Seed initiatorSeed, executorSeed; int j=0; diff --git a/htroot/IndexCreateQueues_p.java b/htroot/IndexCreateQueues_p.java index ba54886fd2..75d169d12a 100644 --- a/htroot/IndexCreateQueues_p.java +++ b/htroot/IndexCreateQueues_p.java @@ -9,7 +9,7 @@ import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.CrawlSwitchboard; diff --git a/htroot/IndexDeletion_p.java b/htroot/IndexDeletion_p.java index 966bf664e4..ad933298ce 100644 --- a/htroot/IndexDeletion_p.java +++ b/htroot/IndexDeletion_p.java @@ -30,12 +30,12 @@ import org.apache.solr.common.SolrDocument; import net.yacy.cora.date.ISO8601Formatter; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.solr.connector.AbstractSolrConnector; import net.yacy.cora.federate.solr.connector.SolrConnector; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.sorting.ScoreMap; import net.yacy.data.WorkTables; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import net.yacy.search.query.QueryModifier; import net.yacy.search.schema.CollectionSchema; @@ -129,7 +129,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (urlStub.startsWith("ftp")) urlStub = "ftp://" + urlStub; else urlStub = "http://" + urlStub; } try { - DigestURI u = new DigestURI(urlStub); + DigestURL u = new DigestURL(urlStub); BlockingQueue dq = defaultConnector.concurrentDocumentsByQuery(CollectionSchema.host_s.getSolrFieldName() + ":\"" + u.getHost() + "\"", 0, 100000000, Long.MAX_VALUE, 100, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName()); SolrDocument doc; try { diff --git a/htroot/IndexFederated_p.java b/htroot/IndexFederated_p.java index 86f2928d35..a478183ec1 100644 --- a/htroot/IndexFederated_p.java +++ b/htroot/IndexFederated_p.java @@ -26,7 +26,7 @@ import org.apache.solr.common.SolrException; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.federate.solr.connector.RemoteSolrConnector; import net.yacy.cora.federate.solr.connector.SolrConnector; import net.yacy.cora.federate.solr.instance.RemoteInstance; diff --git a/htroot/IndexImportOAIPMH_p.java b/htroot/IndexImportOAIPMH_p.java index 77d3dfb3b4..6db80a0c54 100644 --- a/htroot/IndexImportOAIPMH_p.java +++ b/htroot/IndexImportOAIPMH_p.java @@ -31,6 +31,7 @@ import java.util.Set; import java.util.TreeSet; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; @@ -38,7 +39,6 @@ import net.yacy.document.importer.OAIPMHImporter; import net.yacy.document.importer.OAIPMHLoader; import net.yacy.document.importer.ResumptionToken; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -59,9 +59,9 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (post.containsKey("urlstartone")) { String oaipmhurl = post.get("urlstartone"); if (oaipmhurl.indexOf('?',0) < 0) oaipmhurl = oaipmhurl + "?verb=ListRecords&metadataPrefix=oai_dc"; - DigestURI url = null; + DigestURL url = null; try { - url = new DigestURI(oaipmhurl); + url = new DigestURL(oaipmhurl); ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName)); final OAIPMHLoader r = new OAIPMHLoader(sb.loader, url, sb.surrogatesInPath, agent); final ResumptionToken rt = r.getResumptionToken(); @@ -72,7 +72,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea // set next default url try { - final DigestURI nexturl = (rt == null) ? null : rt.resumptionURL(); + final DigestURL nexturl = (rt == null) ? null : rt.resumptionURL(); if (rt != null) prop.put("defaulturl", (nexturl == null) ? "" : nexturl.toNormalform(true)); } catch (final MalformedURLException e) { prop.put("defaulturl", e.getMessage()); @@ -94,9 +94,9 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (post.get("urlstart", "").length() > 0) { final String oaipmhurl = post.get("urlstart", ""); sb.tables.recordAPICall(post, "IndexImportOAIPMH_p.html", WorkTables.TABLE_API_TYPE_CRAWLER, "OAI-PMH import for " + oaipmhurl); - DigestURI url = null; + DigestURL url = null; try { - url = new DigestURI(oaipmhurl); + url = new DigestURL(oaipmhurl); ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName)); final OAIPMHImporter job = new OAIPMHImporter(sb.loader, agent, url); job.start(); @@ -129,12 +129,12 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea final Random r = new Random(System.currentTimeMillis()); // start jobs for the sources - DigestURI url = null; + DigestURL url = null; ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName)); while (!sourceList.isEmpty()) { final String oaipmhurl = sourceList.remove(r.nextInt(sourceList.size())); try { - url = new DigestURI(oaipmhurl); + url = new DigestURL(oaipmhurl); final OAIPMHImporter job = new OAIPMHImporter(sb.loader, agent, url); job.start(); } catch (final MalformedURLException e) { diff --git a/htroot/Load_RSS_p.java b/htroot/Load_RSS_p.java index bab9f79224..dae006c198 100644 --- a/htroot/Load_RSS_p.java +++ b/htroot/Load_RSS_p.java @@ -29,12 +29,13 @@ import java.util.Map; import java.util.regex.Pattern; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.Hit; -import net.yacy.cora.document.RSSFeed; -import net.yacy.cora.document.RSSMessage; -import net.yacy.cora.document.RSSReader; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.feed.Hit; +import net.yacy.cora.document.feed.RSSFeed; +import net.yacy.cora.document.feed.RSSMessage; +import net.yacy.cora.document.feed.RSSReader; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; @@ -48,7 +49,6 @@ import net.yacy.data.WorkTables; import net.yacy.kelondro.blob.Tables; import net.yacy.kelondro.blob.Tables.Row; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; import net.yacy.search.schema.CollectionSchema; @@ -169,9 +169,9 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea ConcurrentLog.logException(e); continue; } - DigestURI url = null; + DigestURL url = null; try { - url = new DigestURI(row.get("url", "")); + url = new DigestURL(row.get("url", "")); } catch (final MalformedURLException e) { ConcurrentLog.warn("Load_RSS", "malformed url '" + row.get("url", "") + "': " + e.getMessage()); continue; @@ -203,7 +203,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea messageurl = row.get("url", ""); if (messageurl.isEmpty()) continue; // get referrer - final DigestURI referrer = sb.getURL(row.get("referrer", "").getBytes()); + final DigestURL referrer = sb.getURL(row.get("referrer", "").getBytes()); // check if feed is registered in scheduler final byte[] api_pk = row.get("api_pk"); final Row r = api_pk == null ? null : sb.tables.select("api", api_pk); @@ -257,9 +257,9 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea boolean record_api = false; - DigestURI url = null; + DigestURL url = null; try { - url = post.containsKey("url") ? new DigestURI(post.get("url", "")) : null; + url = post.containsKey("url") ? new DigestURL(post.get("url", "")) : null; } catch (final MalformedURLException e) { ConcurrentLog.warn("Load_RSS_p", "url not well-formed: '" + post.get("url", "") + "'"); } @@ -280,12 +280,12 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea // index all selected items: description only if (rss != null && post.containsKey("indexSelectedItemContent")) { final RSSFeed feed = rss.getFeed(); - List list = new ArrayList(); + List list = new ArrayList(); Map messages = new HashMap(); loop: for (final Map.Entry entry: post.entrySet()) { if (entry.getValue().startsWith("mark_")) try { final RSSMessage message = feed.getMessage(entry.getValue().substring(5)); - final DigestURI messageurl = new DigestURI(message.getLink()); + final DigestURL messageurl = new DigestURL(message.getLink()); if (RSSLoader.indexTriggered.containsKey(messageurl.hash())) continue loop; messages.put(ASCII.String(messageurl.hash()), message); } catch (final IOException e) { @@ -296,7 +296,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea loop: for (final Map.Entry entry: messages.entrySet()) { try { final RSSMessage message = entry.getValue(); - final DigestURI messageurl = new DigestURI(message.getLink()); + final DigestURL messageurl = new DigestURL(message.getLink()); if (existingurls.get(ASCII.String(messageurl.hash())) != null) continue loop; list.add(messageurl); RSSLoader.indexTriggered.insertIfAbsent(messageurl.hash(), new Date()); @@ -334,10 +334,10 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea prop.putHTML("showitems_ttl", channel == null ? "" : channel.getTTL()); prop.putHTML("showitems_docs", channel == null ? "" : channel.getDocs()); - Map urls = new HashMap(); + Map urls = new HashMap(); for (final Hit item: feed) { try { - final DigestURI messageurl = new DigestURI(item.getLink()); + final DigestURL messageurl = new DigestURL(item.getLink()); urls.put(ASCII.String(messageurl.hash()), messageurl); } catch (final MalformedURLException e) { ConcurrentLog.logException(e); @@ -349,7 +349,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea int i = 0; for (final Hit item: feed) { try { - final DigestURI messageurl = new DigestURI(item.getLink()); + final DigestURL messageurl = new DigestURL(item.getLink()); author = item.getAuthor(); if (author == null) author = item.getCopyright(); pubDate = item.getPubDate(); diff --git a/htroot/MessageSend_p.java b/htroot/MessageSend_p.java index bf4dff509b..f192dd59e3 100644 --- a/htroot/MessageSend_p.java +++ b/htroot/MessageSend_p.java @@ -30,7 +30,7 @@ import java.util.Locale; import java.util.Map; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.RequestHeader; import net.yacy.peers.Protocol; import net.yacy.peers.Seed; diff --git a/htroot/Messages_p.java b/htroot/Messages_p.java index cc5f6d55e0..80aeb46eff 100644 --- a/htroot/Messages_p.java +++ b/htroot/Messages_p.java @@ -32,7 +32,7 @@ import java.util.Locale; import java.util.TreeMap; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.data.MessageBoard; diff --git a/htroot/QuickCrawlLink_p.java b/htroot/QuickCrawlLink_p.java index 85d47c1ee4..159b23cdf8 100644 --- a/htroot/QuickCrawlLink_p.java +++ b/htroot/QuickCrawlLink_p.java @@ -32,7 +32,8 @@ import java.net.MalformedURLException; import java.util.Date; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.Domains; @@ -41,7 +42,6 @@ import net.yacy.cora.util.NumberTools; import net.yacy.crawler.data.CrawlProfile; import net.yacy.crawler.retrieval.Request; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import net.yacy.search.index.Segment; import net.yacy.server.serverObjects; @@ -112,12 +112,12 @@ public static serverObjects respond(final RequestHeader header, final serverObje if (crawlingStart != null) { crawlingStart = crawlingStart.trim(); - try {crawlingStart = new DigestURI(crawlingStart).toNormalform(true);} catch (final MalformedURLException e1) {} + try {crawlingStart = new DigestURL(crawlingStart).toNormalform(true);} catch (final MalformedURLException e1) {} // check if url is proper - DigestURI crawlingStartURL = null; + DigestURL crawlingStartURL = null; try { - crawlingStartURL = new DigestURI(crawlingStart); + crawlingStartURL = new DigestURL(crawlingStart); } catch (final MalformedURLException e) { prop.put("mode_status", "1"); prop.put("mode_code", "1"); diff --git a/htroot/ServerScannerList.java b/htroot/ServerScannerList.java index 9196439648..e5dd62a26e 100644 --- a/htroot/ServerScannerList.java +++ b/htroot/ServerScannerList.java @@ -23,13 +23,13 @@ import java.util.Iterator; import java.util.Map; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.Scanner; import net.yacy.cora.protocol.Scanner.Access; import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.WorkTables; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -54,7 +54,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea // show scancache table prop.put("servertable", 1); String urlString; - DigestURI u; + DigestURL u; table: while (true) { try { int i = 0; diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java index 46ba26bb25..a916a2a2ef 100644 --- a/htroot/SettingsAck_p.java +++ b/htroot/SettingsAck_p.java @@ -35,7 +35,7 @@ import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.Digest; import net.yacy.cora.protocol.Domains; @@ -62,7 +62,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje final Switchboard sb = (Switchboard) env; // get referer for backlink - final MultiProtocolURI referer = header.referer(); + final MultiProtocolURL referer = header.referer(); prop.put("referer", (referer == null) ? "Settings_p.html" : referer.toNormalform(true)); //if (post == null) System.out.println("POST: NULL"); else System.out.println("POST: " + post.toString()); diff --git a/htroot/Supporter.java b/htroot/Supporter.java index c56a3f62d8..391a0c9a5b 100644 --- a/htroot/Supporter.java +++ b/htroot/Supporter.java @@ -31,13 +31,13 @@ import java.util.Iterator; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.order.NaturalOrder; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.sorting.ConcurrentScoreMap; import net.yacy.cora.sorting.ScoreMap; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row.Entry; import net.yacy.peers.NewsDB; @@ -129,7 +129,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje url = row.getPrimaryKeyUTF8().trim(); try { - if (Switchboard.urlBlacklist.isListed(BlacklistType.SURFTIPS, new DigestURI(url, urlhash.getBytes()))) continue; + if (Switchboard.urlBlacklist.isListed(BlacklistType.SURFTIPS, new DigestURL(url, urlhash.getBytes()))) continue; } catch (final MalformedURLException e) { continue; } @@ -247,13 +247,13 @@ record = recordIterator.next(); // add/subtract votes and write record if (entry != null) { try { - urlhash = ASCII.String((new DigestURI(url)).hash()); + urlhash = ASCII.String((new DigestURL(url)).hash()); } catch (final MalformedURLException e) { urlhash = null; } if (urlhash == null) try { - urlhash = ASCII.String((new DigestURI("http://" + url)).hash()); + urlhash = ASCII.String((new DigestURL("http://" + url)).hash()); } catch (final MalformedURLException e) { urlhash = null; } diff --git a/htroot/Surftips.java b/htroot/Surftips.java index 48eb86f84e..0926a87284 100644 --- a/htroot/Surftips.java +++ b/htroot/Surftips.java @@ -31,12 +31,12 @@ import java.util.Iterator; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.order.NaturalOrder; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.sorting.ConcurrentScoreMap; import net.yacy.cora.sorting.ScoreMap; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row.Entry; import net.yacy.peers.NewsDB; @@ -136,7 +136,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje url = row.getPrimaryKeyUTF8().trim(); try{ - if(Switchboard.urlBlacklist.isListed(BlacklistType.SURFTIPS ,new DigestURI(url))) + if(Switchboard.urlBlacklist.isListed(BlacklistType.SURFTIPS ,new DigestURL(url))) continue; }catch(final MalformedURLException e){continue;} title = row.getColUTF8(1); @@ -306,13 +306,13 @@ record = recordIterator.next(); // add/subtract votes and write record if (entry != null) { try { - urlhash = UTF8.String((new DigestURI(url)).hash()); + urlhash = UTF8.String((new DigestURL(url)).hash()); } catch (final MalformedURLException e) { urlhash = null; } if (urlhash == null) try { - urlhash = UTF8.String((new DigestURI("http://"+url)).hash()); + urlhash = UTF8.String((new DigestURL("http://"+url)).hash()); } catch (final MalformedURLException e) { urlhash = null; } diff --git a/htroot/Table_API_p.java b/htroot/Table_API_p.java index 6b3689343d..4533d9bc7f 100644 --- a/htroot/Table_API_p.java +++ b/htroot/Table_API_p.java @@ -29,7 +29,7 @@ import java.util.TreeSet; import java.util.regex.Pattern; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; diff --git a/htroot/Table_YMark_p.java b/htroot/Table_YMark_p.java index d02d935cd2..482d6ca135 100644 --- a/htroot/Table_YMark_p.java +++ b/htroot/Table_YMark_p.java @@ -5,7 +5,7 @@ import java.util.Map; import java.util.regex.Pattern; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; diff --git a/htroot/Tables_p.java b/htroot/Tables_p.java index ec59a8ad79..3e3aa5c425 100644 --- a/htroot/Tables_p.java +++ b/htroot/Tables_p.java @@ -25,7 +25,7 @@ import java.util.Map; import java.util.regex.Pattern; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; diff --git a/htroot/Triple_p.java b/htroot/Triple_p.java index 11cee0daf6..aea54eb256 100644 --- a/htroot/Triple_p.java +++ b/htroot/Triple_p.java @@ -22,10 +22,10 @@ import java.io.IOException; import java.net.MalformedURLException; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.lod.JenaTripleStore; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; import net.yacy.server.http.HTTPDemon; @@ -70,7 +70,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea for (String s: list.split("\n")) { String newurl = s; try { - DigestURI d = new DigestURI (s); + DigestURL d = new DigestURL (s); if (d.getHost().endsWith(".yacy")) { newurl = d.getProtocol()+"://"+HTTPDemon.getAlternativeResolver().resolve(d.getHost())+d.getPath(); diff --git a/htroot/ViewFile.java b/htroot/ViewFile.java index a25bf64fdd..ef41371118 100644 --- a/htroot/ViewFile.java +++ b/htroot/ViewFile.java @@ -36,9 +36,10 @@ import java.util.Map; import java.util.Properties; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.lod.JenaTripleStore; import net.yacy.cora.lod.vocabulary.YaCyMetadata; @@ -54,7 +55,6 @@ import net.yacy.document.WordTokenizer; import net.yacy.document.parser.html.CharacterCoding; import net.yacy.document.parser.html.ImageEntry; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.search.Switchboard; import net.yacy.search.index.Segment; @@ -107,7 +107,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje final String viewMode = post.get("viewMode","parsed"); prop.put("error_vMode-" + viewMode, "1"); - DigestURI url = null; + DigestURL url = null; String descr = ""; final int wordCount = 0; int size = 0; @@ -127,7 +127,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje } // define an url by post parameter - url = new DigestURI(MultiProtocolURI.unescape(urlString)); + url = new DigestURL(MultiProtocolURL.unescape(urlString)); urlHash = ASCII.String(url.hash()); pre = post.getBoolean("pre"); } catch (final MalformedURLException e) {} @@ -185,7 +185,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje } final String[] wordArray = wordArray(post.get("words", null)); - final String ext = MultiProtocolURI.getFileExtension(url.getFileName()); + final String ext = MultiProtocolURL.getFileExtension(url.getFileName()); if (viewMode.equals("plain")) { // TODO: how to handle very large files here ? @@ -311,11 +311,11 @@ public static serverObjects respond(final RequestHeader header, final serverObje prop.put("viewMode", VIEW_MODE_AS_LINKLIST); boolean dark = true; int i = 0; - i += putMediaInfo(prop, wordArray, i, document.getVideolinks(), "video", (i % 2 == 0), document.getAnchors()); - i += putMediaInfo(prop, wordArray, i, document.getAudiolinks(), "audio", (i % 2 == 0), document.getAnchors()); + i += putMediaInfo(prop, wordArray, i, document.getVideolinks(), "video", (i % 2 == 0)); + i += putMediaInfo(prop, wordArray, i, document.getAudiolinks(), "audio", (i % 2 == 0)); dark = (i % 2 == 0); - final Map ts = document.getImages(); + final Map ts = document.getImages(); final Iterator tsi = ts.values().iterator(); ImageEntry entry; while (tsi.hasNext()) { @@ -335,8 +335,8 @@ public static serverObjects respond(final RequestHeader header, final serverObje dark = !dark; i++; } - i += putMediaInfo(prop, wordArray, i, document.getApplinks(), "app", (i % 2 == 0), document.getAnchors()); - i += putMediaInfo(prop, wordArray, i, document.getHyperlinks(), "link", (i % 2 == 0), document.getAnchors()); + i += putMediaInfo(prop, wordArray, i, document.getApplinks(), "app", (i % 2 == 0)); + i += putMediaInfo(prop, wordArray, i, document.getHyperlinks(), "link", (i % 2 == 0)); prop.put("viewMode_links", i); } @@ -439,13 +439,12 @@ private static int putMediaInfo( final serverObjects prop, final String[] wordArray, int c, - final Map media, + final Map media, final String type, - boolean dark, - final Map alllinks) { + boolean dark) { int i = 0; - for (final Map.Entry entry : media.entrySet()) { - final Properties p = alllinks.get(entry.getKey()); + for (final Map.Entry entry : media.entrySet()) { + final Properties p = entry.getKey().getProperties(); final String name = p.getProperty("name", ""); // the name attribute final String rel = p.getProperty("rel", ""); // the rel-attribute final String text = p.getProperty("text", ""); // the text between the tag diff --git a/htroot/ViewImage.java b/htroot/ViewImage.java index f343a01d28..d0e9b12719 100644 --- a/htroot/ViewImage.java +++ b/htroot/ViewImage.java @@ -32,6 +32,7 @@ import java.net.MalformedURLException; import java.util.Map; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.Domains; @@ -41,7 +42,6 @@ import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.URLLicense; import net.yacy.document.ImageParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.workflow.WorkflowProcessor; @@ -74,9 +74,9 @@ public static Image respond(final RequestHeader header, final serverObjects post final String urlLicense = post.get("code", ""); final boolean auth = Domains.isLocalhost(header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "")) || sb.verifyAuthentication(header); // handle access rights - DigestURI url = null; + DigestURL url = null; if ((urlString.length() > 0) && (auth)) try { - url = new DigestURI(urlString); + url = new DigestURL(urlString); } catch (final MalformedURLException e1) { url = null; } @@ -84,7 +84,7 @@ public static Image respond(final RequestHeader header, final serverObjects post if ((url == null) && (urlLicense.length() > 0)) { urlString = URLLicense.releaseLicense(urlLicense); try { - url = new DigestURI(urlString); + url = new DigestURL(urlString); } catch (final MalformedURLException e1) { url = null; urlString = null; diff --git a/htroot/Vocabulary_p.java b/htroot/Vocabulary_p.java index baaeddf871..d660fa10b3 100644 --- a/htroot/Vocabulary_p.java +++ b/htroot/Vocabulary_p.java @@ -26,7 +26,8 @@ import java.util.Map; import java.util.TreeMap; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.lod.vocabulary.DCTerms; import net.yacy.cora.lod.vocabulary.Owl; import net.yacy.cora.lod.vocabulary.Tagging; @@ -35,7 +36,6 @@ import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.LibraryProvider; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.search.Switchboard; import net.yacy.search.index.Segment; @@ -59,8 +59,8 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea // create a vocabulary if (discovername != null && discovername.length() > 0) { String discoverobjectspace = post.get("discoverobjectspace", ""); - MultiProtocolURI discoveruri = null; - if (discoverobjectspace.length() > 0) try {discoveruri = new MultiProtocolURI(discoverobjectspace);} catch (final MalformedURLException e) {} + MultiProtocolURL discoveruri = null; + if (discoverobjectspace.length() > 0) try {discoveruri = new MultiProtocolURL(discoverobjectspace);} catch (final MalformedURLException e) {} if (discoveruri == null) discoverobjectspace = ""; Map table = new TreeMap(); File propFile = LibraryProvider.autotagging.getVocabularyFile(discovername); @@ -72,9 +72,9 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea Segment segment = sb.index; String t; if (!discoverNot) { - Iterator ui = segment.urlSelector(discoveruri, 600000L, 100000); + Iterator ui = segment.urlSelector(discoveruri, 600000L, 100000); while (ui.hasNext()) { - DigestURI u = ui.next(); + DigestURL u = ui.next(); String u0 = u.toNormalform(true); t = ""; if (discoverFromPath) { @@ -131,7 +131,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (post.get("add_new", "").equals("checked") && post.get("newterm", "").length() > 0) { String objectlink = post.get("newobjectlink", ""); if (objectlink.length() > 0) try { - objectlink = new MultiProtocolURI(objectlink).toNormalform(true); + objectlink = new MultiProtocolURL(objectlink).toNormalform(true); } catch (final MalformedURLException e) {} vocabulary.put(post.get("newterm", ""), post.get("newsynonyms", ""), objectlink); } diff --git a/htroot/WatchWebStructure_p.java b/htroot/WatchWebStructure_p.java index a376c7b096..9a7fa087a2 100644 --- a/htroot/WatchWebStructure_p.java +++ b/htroot/WatchWebStructure_p.java @@ -6,11 +6,11 @@ import java.util.Iterator; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.sorting.ReversibleScoreMap; import net.yacy.crawler.CrawlSwitchboard; import net.yacy.crawler.data.CrawlProfile; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -64,7 +64,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea // fix start point if a "www."-prefix would be better if (host != null && !host.startsWith("www")) { - if (sb.webStructure.referencesCount(DigestURI.hosthash6("www." + host)) > sb.webStructure.referencesCount(DigestURI.hosthash6(host))) { + if (sb.webStructure.referencesCount(DigestURL.hosthash6("www." + host)) > sb.webStructure.referencesCount(DigestURL.hosthash6(host))) { host = "www." + host; } } diff --git a/htroot/WebStructurePicture_p.java b/htroot/WebStructurePicture_p.java index 4be6f047fc..ac1f372ac1 100644 --- a/htroot/WebStructurePicture_p.java +++ b/htroot/WebStructurePicture_p.java @@ -33,12 +33,12 @@ import java.util.List; import java.util.Map; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.sorting.ClusteredScoreMap; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.peers.graphics.WebStructureGraph; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; @@ -116,7 +116,7 @@ public static RasterPlotter respond(@SuppressWarnings("unused") final RequestHea for (int i = 0; i < hostlist.length; i++) { String host = hostlist[i]; String hash = null; - try {hash = ASCII.String((new DigestURI("http://" + host)).hash(), 6, 6);} catch (final MalformedURLException e) {ConcurrentLog.logException(e);} + try {hash = ASCII.String((new DigestURL("http://" + host)).hash(), 6, 6);} catch (final MalformedURLException e) {ConcurrentLog.logException(e);} Map.Entry centernode = new AbstractMap.SimpleEntry(hash, host); double angle = 2.0d * i * Math.PI / hostlist.length; if (hostlist.length == 3) angle -= Math.PI / 2; diff --git a/htroot/Wiki.java b/htroot/Wiki.java index 15b52e6bd9..a3aa6b1f27 100644 --- a/htroot/Wiki.java +++ b/htroot/Wiki.java @@ -38,7 +38,7 @@ import java.util.Locale; import java.util.Map; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; diff --git a/htroot/YMarks.java b/htroot/YMarks.java index 7b3b99d1af..1e4efa5c18 100644 --- a/htroot/YMarks.java +++ b/htroot/YMarks.java @@ -1,7 +1,7 @@ import java.io.IOException; import java.util.Iterator; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; diff --git a/htroot/api/bookmarks/posts/delete_p.java b/htroot/api/bookmarks/posts/delete_p.java index 6cb85617d2..1e65bf1113 100644 --- a/htroot/api/bookmarks/posts/delete_p.java +++ b/htroot/api/bookmarks/posts/delete_p.java @@ -1,9 +1,9 @@ import java.net.MalformedURLException; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -23,7 +23,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje return prop; } try { - if (post.containsKey("url") && switchboard.bookmarksDB.removeBookmark(ASCII.String((new DigestURI(post.get("url", "nourl"))).hash()))) { + if (post.containsKey("url") && switchboard.bookmarksDB.removeBookmark(ASCII.String((new DigestURL(post.get("url", "nourl"))).hash()))) { prop.put("result", "1"); } else if (post.containsKey("urlhash") && switchboard.bookmarksDB.removeBookmark(post.get("urlhash", "nohash"))) { prop.put("result", "1"); diff --git a/htroot/api/citation.java b/htroot/api/citation.java index e77e44746a..2d76f3fc9c 100644 --- a/htroot/api/citation.java +++ b/htroot/api/citation.java @@ -30,12 +30,12 @@ import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.solr.connector.SolrConnector; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.sorting.OrderedScoreMap; import net.yacy.document.SentenceReader; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import net.yacy.search.index.Segment; import net.yacy.search.schema.CollectionSchema; @@ -56,7 +56,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea prop.put("citations", 0); prop.put("sentences", 0); - DigestURI uri = null; + DigestURL uri = null; String url = ""; String hash = ""; int ch = 10; @@ -81,7 +81,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea if (url.length() > 0) { try { - uri = new DigestURI(url, null); + uri = new DigestURL(url, null); hash = ASCII.String(uri.hash()); } catch (final MalformedURLException e) {} } @@ -118,7 +118,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea // for each line make a statistic about the number of occurrences somewhere else OrderedScoreMap scores = new OrderedScoreMap(null); // accumulates scores for citating urls - LinkedHashMap> sentenceOcc = new LinkedHashMap>(); + LinkedHashMap> sentenceOcc = new LinkedHashMap>(); for (String sentence: sentences) { if (sentence == null || sentence.length() < 40) { // do not count the very short sentences @@ -130,12 +130,12 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea SolrDocumentList doclist = connector.getDocumentListByQuery("text_t:\"" + sentence + "\"", 0, 100, CollectionSchema.sku.getSolrFieldName()); int count = (int) doclist.getNumFound(); if (count > 0) { - Set list = new TreeSet(); + Set list = new TreeSet(); for (SolrDocument d: doclist) { String u = (String) d.getFieldValue(CollectionSchema.sku.getSolrFieldName()); if (u == null || u.equals(url)) continue; scores.inc(u); - try {list.add(new DigestURI(u, null));} catch (final MalformedURLException e) {} + try {list.add(new DigestURL(u, null));} catch (final MalformedURLException e) {} } sentenceOcc.put(sentence, list); } @@ -147,13 +147,13 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea // iterate the sentences int i = 0; - for (Map.Entry> se: sentenceOcc.entrySet()) { + for (Map.Entry> se: sentenceOcc.entrySet()) { prop.put("sentences_" + i + "_dt", i); StringBuilder dd = new StringBuilder(se.getKey()); - Set app = se.getValue(); + Set app = se.getValue(); if (app != null && app.size() > 0) { dd.append("
appears in:"); - for (DigestURI u: app) { + for (DigestURL u: app) { if (u != null) { dd.append(" ").append(u.getHost()).append(""); } @@ -168,12 +168,12 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea i = 0; for (String u: scores.keyList(false)) { try { - DigestURI uu = new DigestURI(u, null); + DigestURL uu = new DigestURL(u, null); prop.put("citations_" + i + "_dt", "" + u + ""); StringBuilder dd = new StringBuilder(); dd.append("makes ").append(Integer.toString(scores.get(u))).append(" citations: of ").append(url); - for (Map.Entry> se: sentenceOcc.entrySet()) { - Set occurls = se.getValue(); + for (Map.Entry> se: sentenceOcc.entrySet()) { + Set occurls = se.getValue(); if (occurls != null && occurls.contains(uu)) dd.append("
").append(se.getKey()).append(""); } prop.put("citations_" + i + "_dd", dd.toString()); @@ -187,7 +187,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea for (String u: scores.keyList(false)) { if (scores.get(u) < ch) continue; try { - DigestURI uu = new DigestURI(u, null); + DigestURL uu = new DigestURL(u, null); if (uu.getOrganization().equals(uri.getOrganization())) continue; prop.put("similar_links_" + i + "_url", u); i++; diff --git a/htroot/api/feed.java b/htroot/api/feed.java index 84bc5e49a2..e4b5b7f9cd 100644 --- a/htroot/api/feed.java +++ b/htroot/api/feed.java @@ -3,8 +3,8 @@ import java.util.Date; import java.util.List; -import net.yacy.cora.document.RSSFeed; -import net.yacy.cora.document.RSSMessage; +import net.yacy.cora.document.feed.RSSFeed; +import net.yacy.cora.document.feed.RSSMessage; import net.yacy.cora.protocol.RequestHeader; import net.yacy.peers.EventChannel; import net.yacy.search.Switchboard; diff --git a/htroot/api/getpageinfo.java b/htroot/api/getpageinfo.java index e2d41313d9..eb6cd052b5 100644 --- a/htroot/api/getpageinfo.java +++ b/htroot/api/getpageinfo.java @@ -26,19 +26,21 @@ import java.io.IOException; import java.net.MalformedURLException; +import java.util.Collection; import java.util.Set; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.AnchorURL; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.robots.RobotsTxtEntry; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; @@ -90,9 +92,9 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea url = "http://" + url; } if (actions.indexOf("title",0) >= 0) { - DigestURI u = null; + DigestURL u = null; try { - u = new DigestURI(url); + u = new DigestURL(url); } catch (final MalformedURLException e) { ConcurrentLog.logException(e); } @@ -129,11 +131,11 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea prop.putXML("lang", (languages == null || languages.size() == 0) ? "unknown" : languages.iterator().next()); // get links and put them into a semicolon-separated list - final Set uris = scraper.getAnchors().keySet(); + final Collection uris = scraper.getAnchors(); final StringBuilder links = new StringBuilder(uris.size() * 80); final StringBuilder filter = new StringBuilder(uris.size() * 40); count = 0; - for (final DigestURI uri: uris) { + for (final DigestURL uri: uris) { if (uri == null) continue; links.append(';').append(uri.toNormalform(true)); filter.append('|').append(uri.getProtocol()).append("://").append(uri.getHost()).append(".*"); @@ -147,7 +149,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea } if (actions.indexOf("robots",0) >= 0) { try { - final DigestURI theURL = new DigestURI(url); + final DigestURL theURL = new DigestURL(url); // determine if crawling of the current URL is allowed RobotsTxtEntry robotsEntry = sb.robots.getEntry(theURL, agent); @@ -155,7 +157,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea prop.putHTML("robotsInfo", robotsEntry == null ? "" : robotsEntry.getInfo()); // get the sitemap URL of the domain - final MultiProtocolURI sitemapURL = robotsEntry == null ? null : robotsEntry.getSitemap(); + final MultiProtocolURL sitemapURL = robotsEntry == null ? null : robotsEntry.getSitemap(); prop.putXML("sitemap", sitemapURL == null ? "" : sitemapURL.toString()); } catch (final MalformedURLException e) { ConcurrentLog.logException(e); @@ -163,7 +165,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea } if (actions.indexOf("oai",0) >= 0) { try { - final DigestURI theURL = new DigestURI(url + final DigestURL theURL = new DigestURL(url + "?verb=Identify"); final String oairesult = checkOAI(theURL.toString()); diff --git a/htroot/api/getpageinfo_p.java b/htroot/api/getpageinfo_p.java index 7b35f7e438..6981d93971 100644 --- a/htroot/api/getpageinfo_p.java +++ b/htroot/api/getpageinfo_p.java @@ -26,19 +26,21 @@ import java.io.IOException; import java.net.MalformedURLException; +import java.util.Collection; import java.util.Set; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.AnchorURL; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.robots.RobotsTxtEntry; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; @@ -88,9 +90,9 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea url = "http://" + url; } if (actions.indexOf("title",0) >= 0) { - DigestURI u = null; + DigestURL u = null; try { - u = new DigestURI(url); + u = new DigestURL(url); } catch (final MalformedURLException e) { ConcurrentLog.logException(e); } @@ -128,11 +130,11 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea prop.putXML("lang", (languages == null || languages.size() == 0) ? "unknown" : languages.iterator().next()); // get links and put them into a semicolon-separated list - final Set uris = scraper.getAnchors().keySet(); + final Collection uris = scraper.getAnchors(); final StringBuilder links = new StringBuilder(uris.size() * 80); final StringBuilder filter = new StringBuilder(uris.size() * 40); count = 0; - for (final DigestURI uri: uris) { + for (final DigestURL uri: uris) { if (uri == null) continue; links.append(';').append(uri.toNormalform(true)); filter.append('|').append(uri.getProtocol()).append("://").append(uri.getHost()).append(".*"); @@ -146,7 +148,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea } if (actions.indexOf("robots",0) >= 0) { try { - final DigestURI theURL = new DigestURI(url); + final DigestURL theURL = new DigestURL(url); // determine if crawling of the current URL is allowed ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName)); @@ -156,7 +158,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea prop.putHTML("robotsInfo", robotsEntry == null ? "" : robotsEntry.getInfo()); // get the sitemap URL of the domain - final MultiProtocolURI sitemapURL = robotsEntry == null ? null : robotsEntry.getSitemap(); + final MultiProtocolURL sitemapURL = robotsEntry == null ? null : robotsEntry.getSitemap(); prop.putXML("sitemap", sitemapURL == null ? "" : sitemapURL.toString()); } catch (final MalformedURLException e) { ConcurrentLog.logException(e); @@ -164,7 +166,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea } if (actions.indexOf("oai",0) >= 0) { try { - final DigestURI theURL = new DigestURI(url + final DigestURL theURL = new DigestURL(url + "?verb=Identify"); final String oairesult = checkOAI(theURL.toString()); diff --git a/htroot/api/table_p.java b/htroot/api/table_p.java index 819172fd46..2df19a36d2 100644 --- a/htroot/api/table_p.java +++ b/htroot/api/table_p.java @@ -24,7 +24,7 @@ import java.util.Map; import java.util.regex.Pattern; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; diff --git a/htroot/api/termlist_p.java b/htroot/api/termlist_p.java index 6a342c8fba..4d420a6501 100644 --- a/htroot/api/termlist_p.java +++ b/htroot/api/termlist_p.java @@ -25,7 +25,7 @@ import java.util.ArrayList; import java.util.Iterator; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.sorting.Rating; import net.yacy.cora.util.ConcurrentLog; diff --git a/htroot/api/webstructure.java b/htroot/api/webstructure.java index 94731fadee..e8a9efde4a 100644 --- a/htroot/api/webstructure.java +++ b/htroot/api/webstructure.java @@ -29,14 +29,14 @@ import java.util.Map; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.kelondro.data.citation.CitationReference; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.rwi.IndexCell; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.peers.graphics.WebStructureGraph; @@ -56,7 +56,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje prop.put("citations", 0); boolean authenticated = sb.adminAuthenticated(header) >= 2; if (about != null) { - DigestURI url = null; + DigestURL url = null; byte[] urlhash = null; String hosthash = null; if (about.length() == 6 && Base64Order.enhancedCoder.wellformed(ASCII.getBytes(about))) { @@ -68,7 +68,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje } else if (authenticated && about.length() > 0) { // consider "about" as url or hostname try { - url = new DigestURI(about.indexOf("://") >= 0 ? about : "http://" + about); // accept also domains + url = new DigestURL(about.indexOf("://") >= 0 ? about : "http://" + about); // accept also domains urlhash = url.hash(); hosthash = ASCII.String(urlhash, 6, 6); } catch (final MalformedURLException e) { @@ -111,18 +111,18 @@ public static serverObjects respond(final RequestHeader header, final serverObje prop.put("references_documents_0_urle", url == null ? 0 : 1); if (url != null) prop.putXML("references_documents_0_urle_url", url.toNormalform(true)); int d = 0; - Iterator i = scraper.inboundLinks().iterator(); + Iterator i = scraper.inboundLinks().keySet().iterator(); while (i.hasNext()) { - DigestURI refurl = i.next(); + DigestURL refurl = i.next(); byte[] refhash = refurl.hash(); prop.putXML("references_documents_0_anchors_" + d + "_url", refurl.toNormalform(true)); prop.put("references_documents_0_anchors_" + d + "_hash", refhash); prop.put("references_documents_0_anchors_" + d + "_outbound", 0); d++; } - i = scraper.outboundLinks().iterator(); + i = scraper.outboundLinks().keySet().iterator(); while (i.hasNext()) { - DigestURI refurl = i.next(); + DigestURL refurl = i.next(); byte[] refhash = refurl.hash(); prop.putXML("references_documents_0_anchors_" + d + "_url", refurl.toNormalform(true)); prop.put("references_documents_0_anchors_" + d + "_hash", refhash); @@ -158,7 +158,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje while (i.hasNext()) { CitationReference cr = i.next(); byte[] refhash = cr.urlhash(); - DigestURI refurl = authenticated ? sb.getURL(refhash) : null; + DigestURL refurl = authenticated ? sb.getURL(refhash) : null; prop.put("citations_documents_0_anchors_" + d + "_urle", refurl == null ? 0 : 1); if (refurl != null) prop.putXML("citations_documents_0_anchors_" + d + "_urle_url", refurl.toNormalform(true)); prop.put("citations_documents_0_anchors_" + d + "_urle_hash", refhash); diff --git a/htroot/api/yacydoc.java b/htroot/api/yacydoc.java index 2470049fbd..e394fe1a1d 100644 --- a/htroot/api/yacydoc.java +++ b/htroot/api/yacydoc.java @@ -30,13 +30,13 @@ import java.util.Iterator; import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.lod.JenaTripleStore; import net.yacy.cora.lod.vocabulary.YaCyMetadata; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader.FileType; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.word.Word; import net.yacy.search.Switchboard; @@ -90,7 +90,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje if (urlstring.length() > 0 && urlhash.isEmpty()) { try { - final DigestURI url = new DigestURI(urlstring); + final DigestURL url = new DigestURL(urlstring); urlhash = ASCII.String(url.hash()); } catch (final MalformedURLException e) { ConcurrentLog.logException(e); diff --git a/htroot/api/ymarks/add_ymark.java b/htroot/api/ymarks/add_ymark.java index c6ce581c78..b001d0a59d 100644 --- a/htroot/api/ymarks/add_ymark.java +++ b/htroot/api/ymarks/add_ymark.java @@ -1,5 +1,6 @@ import java.io.IOException; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; @@ -8,7 +9,6 @@ import net.yacy.data.ymark.YMarkTables; import net.yacy.data.ymark.YMarkUtil; import net.yacy.document.Parser.Failure; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -34,7 +34,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje if (post.containsKey("urlHash")) { final String urlHash = post.get("urlHash",YMarkUtil.EMPTY_STRING); - final DigestURI url = sb.index.fulltext().getURL(urlHash.getBytes()); + final DigestURL url = sb.index.fulltext().getURL(urlHash.getBytes()); final String folders = post.get(YMarkEntry.BOOKMARK.FOLDERS.key(),YMarkEntry.BOOKMARK.FOLDERS.deflt()); final String tags = post.get(YMarkEntry.BOOKMARK.TAGS.key(),YMarkUtil.EMPTY_STRING); try { diff --git a/htroot/api/ymarks/get_metadata.java b/htroot/api/ymarks/get_metadata.java index aaae81ecc8..c161a1e1eb 100644 --- a/htroot/api/ymarks/get_metadata.java +++ b/htroot/api/ymarks/get_metadata.java @@ -4,6 +4,7 @@ import java.util.Iterator; import java.util.regex.Pattern; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; @@ -16,7 +17,6 @@ import net.yacy.data.ymark.YMarkUtil; import net.yacy.document.Document; import net.yacy.document.Parser.Failure; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -49,7 +49,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje } try { - final YMarkMetadata meta = new YMarkMetadata(new DigestURI(url), sb.index); + final YMarkMetadata meta = new YMarkMetadata(new DigestURL(url), sb.index); ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName)); final Document document = meta.loadDocument(sb.loader, agent); final EnumMap metadata = meta.loadMetadata(); diff --git a/htroot/api/ymarks/get_treeview.java b/htroot/api/ymarks/get_treeview.java index 5ef79a4e65..300cc3a151 100644 --- a/htroot/api/ymarks/get_treeview.java +++ b/htroot/api/ymarks/get_treeview.java @@ -7,7 +7,8 @@ import java.util.regex.Pattern; import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; @@ -23,7 +24,6 @@ import net.yacy.document.Document; import net.yacy.document.Parser.Failure; import net.yacy.kelondro.blob.Tables; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -212,7 +212,7 @@ else if(foldername[n].equals("Crawl Start")) } } else if (isAutoTagger || isMetadata || isURLdb || isCrawlStart) { try { - final YMarkMetadata meta = new YMarkMetadata(new DigestURI(post.get(ROOT).substring(2)), sb.index); + final YMarkMetadata meta = new YMarkMetadata(new DigestURL(post.get(ROOT).substring(2)), sb.index); ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName)); final Document document = meta.loadDocument(sb.loader, agent); final TreeMap tags = sb.tables.bookmarks.getTags(bmk_user); diff --git a/htroot/api/ymarks/get_xbel.java b/htroot/api/ymarks/get_xbel.java index 3ed5c6e05a..c478095465 100644 --- a/htroot/api/ymarks/get_xbel.java +++ b/htroot/api/ymarks/get_xbel.java @@ -2,7 +2,7 @@ import java.util.HashSet; import java.util.Iterator; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.UserDB; diff --git a/htroot/api/ymarks/get_ymark.java b/htroot/api/ymarks/get_ymark.java index 5560f20102..e2d2956cb9 100644 --- a/htroot/api/ymarks/get_ymark.java +++ b/htroot/api/ymarks/get_ymark.java @@ -3,7 +3,7 @@ import java.util.Iterator; import java.util.regex.Pattern; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.UserDB; diff --git a/htroot/api/ymarks/import_ymark.java b/htroot/api/ymarks/import_ymark.java index 8a21f80708..d10e698c39 100644 --- a/htroot/api/ymarks/import_ymark.java +++ b/htroot/api/ymarks/import_ymark.java @@ -12,7 +12,7 @@ import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; diff --git a/htroot/cytag.java b/htroot/cytag.java index 5d29544138..d3b6eaded6 100644 --- a/htroot/cytag.java +++ b/htroot/cytag.java @@ -28,8 +28,9 @@ import java.awt.Image; import java.io.File; import java.io.IOException; + import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.document.ImageParser; @@ -45,7 +46,7 @@ public class cytag { public static Image respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final Switchboard sb = (Switchboard)env; - final MultiProtocolURI referer = header.referer(); + final MultiProtocolURL referer = header.referer(); // harvest request information StringBuilder connect = new StringBuilder(); diff --git a/htroot/gsa/searchresult.java b/htroot/gsa/searchresult.java index 6ccf143eee..1a145cf94b 100644 --- a/htroot/gsa/searchresult.java +++ b/htroot/gsa/searchresult.java @@ -25,7 +25,7 @@ import java.util.Date; import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.federate.solr.Ranking; import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector; import net.yacy.cora.federate.solr.responsewriter.GSAResponseWriter; diff --git a/htroot/interaction/GetRDF.java b/htroot/interaction/GetRDF.java index 4c704c7df3..679770db40 100644 --- a/htroot/interaction/GetRDF.java +++ b/htroot/interaction/GetRDF.java @@ -32,7 +32,7 @@ import java.io.ByteArrayOutputStream; import java.io.UnsupportedEncodingException; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.lod.JenaTripleStore; import net.yacy.cora.protocol.RequestHeader; import net.yacy.interaction.Interaction; diff --git a/htroot/mediawiki_p.java b/htroot/mediawiki_p.java index e2d82500b3..3a6bf3864b 100644 --- a/htroot/mediawiki_p.java +++ b/htroot/mediawiki_p.java @@ -27,7 +27,7 @@ import java.io.File; import java.io.IOException; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.RequestHeader; import net.yacy.document.importer.MediawikiImporter; import net.yacy.search.Switchboard; diff --git a/htroot/rct_p.java b/htroot/rct_p.java index b8c7efb66f..8c05077d5b 100644 --- a/htroot/rct_p.java +++ b/htroot/rct_p.java @@ -29,11 +29,11 @@ import java.util.Date; import java.util.Iterator; -import net.yacy.cora.document.Hit; -import net.yacy.cora.document.RSSFeed; +import net.yacy.cora.document.feed.Hit; +import net.yacy.cora.document.feed.RSSFeed; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.RequestHeader; import net.yacy.crawler.retrieval.Request; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.peers.DHTSelection; import net.yacy.peers.Protocol; import net.yacy.peers.Seed; @@ -58,15 +58,15 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea //System.out.println("URL=" + item.getLink() + ", desc=" + item.getDescription() + ", pubDate=" + item.getPubDate()); // put url on remote crawl stack - DigestURI url; + DigestURL url; try { - url = new DigestURI(item.getLink()); + url = new DigestURL(item.getLink()); } catch (final MalformedURLException e) { url = null; } Date loaddate; loaddate = item.getPubDate(); - final DigestURI referrer = null; // referrer needed! + final DigestURL referrer = null; // referrer needed! final String urlRejectReason = sb.crawlStacker.urlInAcceptedDomain(url); if (urlRejectReason == null) { // stack url @@ -101,7 +101,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea * @param url * @return */ - private static String urlToString(final DigestURI url) { + private static String urlToString(final DigestURL url) { return (url == null ? "null" : url.toNormalform(true)); } diff --git a/htroot/sharedBlacklist_p.java b/htroot/sharedBlacklist_p.java index 86430b9800..43c56b6b08 100644 --- a/htroot/sharedBlacklist_p.java +++ b/htroot/sharedBlacklist_p.java @@ -38,14 +38,14 @@ import java.util.List; import java.util.Set; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; import net.yacy.data.ListManager; import net.yacy.data.list.ListAccumulator; import net.yacy.data.list.XMLBlacklistImporter; import net.yacy.document.parser.html.CharacterCoding; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.peers.Seed; import net.yacy.repository.Blacklist; @@ -137,7 +137,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea // download the blacklist try { // get List - final DigestURI u = new DigestURI(downloadURLOld); + final DigestURL u = new DigestURL(downloadURLOld); otherBlacklist = FileUtils.strings(u.get(agent)); } catch (final Exception e) { @@ -155,7 +155,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea prop.putHTML("page_source", downloadURL); try { - final DigestURI u = new DigestURI(downloadURL); + final DigestURL u = new DigestURL(downloadURL); otherBlacklist = FileUtils.strings(u.get(agent)); } catch (final Exception e) { prop.put("status", STATUS_URL_PROBLEM); diff --git a/htroot/solr/select.java b/htroot/solr/select.java index 3d5a1e402b..69df3e1101 100644 --- a/htroot/solr/select.java +++ b/htroot/solr/select.java @@ -27,7 +27,7 @@ import javax.servlet.ServletException; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.federate.solr.Ranking; import net.yacy.cora.federate.solr.SolrServlet; import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector; diff --git a/htroot/yacy/crawlReceipt.java b/htroot/yacy/crawlReceipt.java index d83aa38d69..a3a3318e65 100644 --- a/htroot/yacy/crawlReceipt.java +++ b/htroot/yacy/crawlReceipt.java @@ -29,7 +29,7 @@ import java.io.IOException; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.data.ResultURLs; diff --git a/htroot/yacy/idx.java b/htroot/yacy/idx.java index e5c1afd04e..7662a6b084 100644 --- a/htroot/yacy/idx.java +++ b/htroot/yacy/idx.java @@ -24,7 +24,7 @@ import java.util.Iterator; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.protocol.RequestHeader; import net.yacy.kelondro.rwi.ReferenceContainer; import net.yacy.kelondro.rwi.ReferenceContainerCache; diff --git a/htroot/yacy/message.java b/htroot/yacy/message.java index 83cbde5325..3544c41865 100644 --- a/htroot/yacy/message.java +++ b/htroot/yacy/message.java @@ -35,7 +35,7 @@ import java.util.Date; import java.util.Locale; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 456b4a898c..77316289d7 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -36,10 +36,11 @@ import java.util.TreeMap; import java.util.TreeSet; -import net.yacy.cora.document.ASCII; import net.yacy.cora.document.analysis.Classification; import net.yacy.cora.document.analysis.Classification.ContentDomain; -import net.yacy.cora.document.RSSMessage; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.feed.RSSMessage; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; @@ -50,7 +51,6 @@ import net.yacy.cora.storage.HandleSet; import net.yacy.cora.util.ByteBuffer; import net.yacy.cora.util.SpaceExceededException; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.data.word.WordReferenceFactory; import net.yacy.kelondro.data.word.WordReferenceRow; @@ -241,7 +241,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje null, false, null, - DigestURI.TLD_any_zone_filter, + DigestURL.TLD_any_zone_filter, client, false, indexSegment, @@ -305,7 +305,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje constraint, false, null, - DigestURI.TLD_any_zone_filter, + DigestURL.TLD_any_zone_filter, client, false, sb.index, diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java index 436ed9ddc8..bd6f18e67c 100644 --- a/htroot/yacy/transferRWI.java +++ b/htroot/yacy/transferRWI.java @@ -32,9 +32,9 @@ import java.util.Iterator; import java.util.Set; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.RSSMessage; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.feed.RSSMessage; import net.yacy.cora.federate.yacy.Distribution; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; diff --git a/htroot/yacy/transferURL.java b/htroot/yacy/transferURL.java index a010a64b80..f28fe9137e 100644 --- a/htroot/yacy/transferURL.java +++ b/htroot/yacy/transferURL.java @@ -33,8 +33,8 @@ import java.util.Set; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.RSSMessage; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.feed.RSSMessage; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.data.ResultURLs; diff --git a/htroot/yacy/urls.java b/htroot/yacy/urls.java index 3fa8cdd000..f14e712256 100644 --- a/htroot/yacy/urls.java +++ b/htroot/yacy/urls.java @@ -28,12 +28,12 @@ import java.util.Date; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.RequestHeader; import net.yacy.crawler.data.NoticedURL; import net.yacy.crawler.data.ZURL.FailCategory; import net.yacy.crawler.retrieval.Request; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.peers.Protocol; import net.yacy.search.Switchboard; @@ -65,7 +65,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea final long timeout = System.currentTimeMillis() + maxTime; int c = 0; Request entry; - DigestURI referrer; + DigestURL referrer; while ((maxCount > 0) && (System.currentTimeMillis() < timeout) && (sb.crawlQueues.noticeURL.stackSize(stackType) > 0)) { @@ -112,7 +112,7 @@ public static serverObjects respond(@SuppressWarnings("unused") final RequestHea final int count = urlhashes.length() / 12; int c = 0; URIMetadataNode entry; - DigestURI referrer; + DigestURL referrer; for (int i = 0; i < count; i++) { entry = sb.index.fulltext().getMetadata(ASCII.getBytes(urlhashes.substring(12 * i, 12 * (i + 1)))); if (entry == null) continue; diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 80e55f6300..6e028f53ed 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -42,9 +42,9 @@ import net.yacy.cora.document.analysis.Classification; import net.yacy.cora.document.analysis.Classification.ContentDomain; -import net.yacy.cora.document.RSSMessage; -import net.yacy.cora.document.UTF8; - +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.feed.RSSMessage; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.opensearch.OpenSearchConnector; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.geo.GeoLocation; @@ -62,7 +62,6 @@ import net.yacy.document.Document; import net.yacy.document.LibraryProvider; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.util.Bitfield; import net.yacy.kelondro.util.Formatter; @@ -606,7 +605,7 @@ public static serverObjects respond( return prop; } final String bookmarkHash = post.get("bookmarkref", ""); // urlhash - final DigestURI url = indexSegment.fulltext().getURL(UTF8.getBytes(bookmarkHash)); + final DigestURL url = indexSegment.fulltext().getURL(UTF8.getBytes(bookmarkHash)); if ( url != null ) { try { sb.tables.bookmarks.createBookmark( @@ -658,8 +657,8 @@ public static serverObjects respond( clustersearch && global ? QueryParams.Searchdom.CLUSTER : (global && indexReceiveGranted ? QueryParams.Searchdom.GLOBAL : QueryParams.Searchdom.LOCAL), constraint, true, - DigestURI.hosthashess(sb.getConfig("search.excludehosth", "")), - DigestURI.TLD_any_zone_filter, + DigestURL.hosthashess(sb.getConfig("search.excludehosth", "")), + DigestURL.TLD_any_zone_filter, client, authenticated, indexSegment, diff --git a/htroot/yacysearch_location.java b/htroot/yacysearch_location.java index 3112a7aafe..7a55b606dd 100644 --- a/htroot/yacysearch_location.java +++ b/htroot/yacysearch_location.java @@ -23,7 +23,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; -import net.yacy.cora.document.RSSMessage; +import net.yacy.cora.document.feed.RSSMessage; import net.yacy.cora.federate.opensearch.SRURSSConnector; import net.yacy.cora.geo.GeoLocation; import net.yacy.cora.protocol.Domains; diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index f37897e14d..570dc5d2ac 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -28,11 +28,12 @@ import java.util.List; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.RSSMessage; import net.yacy.cora.document.analysis.Classification; import net.yacy.cora.document.analysis.Classification.ContentDomain; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.feed.RSSMessage; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; @@ -41,7 +42,6 @@ import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.data.Cache; import net.yacy.data.URLLicense; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.Formatter; import net.yacy.peers.NewsPool; import net.yacy.peers.Seed; @@ -116,13 +116,13 @@ public static serverObjects respond(final RequestHeader header, final serverObje final ResultEntry result = theSearch.oneResult(item, timeout); if (result == null) return prop; // no content final String resultUrlstring = result.urlstring(); - final DigestURI resultURL = result.url(); + final DigestURL resultURL = result.url(); final String target = sb.getConfig(resultUrlstring.matches(target_special_pattern) ? SwitchboardConstants.SEARCH_TARGET_SPECIAL : SwitchboardConstants.SEARCH_TARGET_DEFAULT, "_self"); final int port = resultURL.getPort(); - DigestURI faviconURL = null; + DigestURL faviconURL = null; if ((fileType == FileType.HTML || fileType == FileType.JSON) && !sb.isIntranetMode()) try { - faviconURL = new DigestURI(resultURL.getProtocol() + "://" + resultURL.getHost() + ((port != -1) ? (":" + port) : "") + "/favicon.ico"); + faviconURL = new DigestURL(resultURL.getProtocol() + "://" + resultURL.getHost() + ((port != -1) ? (":" + port) : "") + "/favicon.ico"); } catch (final MalformedURLException e1) { ConcurrentLog.logException(e1); faviconURL = null; @@ -166,7 +166,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje // check if url is allowed to view if (sb.getConfig("proxyURL.rewriteURLs", "all").equals("domainlist")) { try { - if (sb.crawlStacker.urlInAcceptedDomain(new DigestURI (modifyURL)) == null) { + if (sb.crawlStacker.urlInAcceptedDomain(new DigestURL (modifyURL)) == null) { modifyURL = "./proxy.html?url="+modifyURL; } } catch (final MalformedURLException e) { @@ -177,7 +177,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje if (sb.getConfig("proxyURL.rewriteURLs", "all").equals("yacy")) { try { - if ((new DigestURI (modifyURL).getHost().endsWith(".yacy"))) { + if ((new DigestURL (modifyURL).getHost().endsWith(".yacy"))) { modifyURL = "./proxy.html?url="+modifyURL; } } catch (final MalformedURLException e) { @@ -245,7 +245,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje prop.put("content_heuristic_name", heuristic.heuristicName); } EventTracker.update(EventTracker.EClass.SEARCH, new ProfilingGraph.EventSearch(theSearch.query.id(true), SearchEventType.FINALIZATION, "" + item, 0, 0), false); - final String ext = MultiProtocolURI.getFileExtension(resultFileName).toLowerCase(); + final String ext = MultiProtocolURL.getFileExtension(resultFileName).toLowerCase(); if (ext.equals("png") || ext.equals("jpg") || ext.equals("gif")) { final String license = URLLicense.aquireLicense(resultURL); prop.put("content_code", license); diff --git a/htroot/yacysearchtrailer.java b/htroot/yacysearchtrailer.java index 3214e666d7..ac576f339a 100644 --- a/htroot/yacysearchtrailer.java +++ b/htroot/yacysearchtrailer.java @@ -27,8 +27,8 @@ import java.util.Iterator; import java.util.Map; -import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.analysis.Classification; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.sorting.ScoreMap; @@ -346,7 +346,7 @@ public static serverObjects respond(final RequestHeader header, final serverObje if (count == 0) { break; } - nav = "%2Fvocabulary%2F" + navname + "%2F" + MultiProtocolURI.escape(Tagging.encodePrintname(name)).toString(); + nav = "%2Fvocabulary%2F" + navname + "%2F" + MultiProtocolURL.escape(Tagging.encodePrintname(name)).toString(); queryStringForUrl = theSearch.query.getQueryGoal().getOriginalQueryString(true); p = queryStringForUrl.indexOf(nav); if (p < 0) { diff --git a/source/net/yacy/contentcontrol/SMWListSyncThread.java b/source/net/yacy/contentcontrol/SMWListSyncThread.java index 97ef5989d8..b4d2111550 100644 --- a/source/net/yacy/contentcontrol/SMWListSyncThread.java +++ b/source/net/yacy/contentcontrol/SMWListSyncThread.java @@ -5,7 +5,7 @@ import java.net.MalformedURLException; import java.net.URL; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.cora.util.ConcurrentLog; diff --git a/source/net/yacy/cora/document/analysis/Classification.java b/source/net/yacy/cora/document/analysis/Classification.java index 61bfd5e319..470c1e98fe 100644 --- a/source/net/yacy/cora/document/analysis/Classification.java +++ b/source/net/yacy/cora/document/analysis/Classification.java @@ -28,7 +28,7 @@ import java.util.Properties; import java.util.Set; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.util.CommonPattern; public class Classification { @@ -200,11 +200,11 @@ public static String ext2mime(final String ext, final String dfltMime) { return ext == null ? "application/octet-stream" : mimeTable.getProperty(ext.toLowerCase(), dfltMime); } - public static String url2mime(final MultiProtocolURI url, final String dfltMime) { - return url == null ? "application/octet-stream" : ext2mime(MultiProtocolURI.getFileExtension(url.getFileName()), dfltMime); + public static String url2mime(final MultiProtocolURL url, final String dfltMime) { + return url == null ? "application/octet-stream" : ext2mime(MultiProtocolURL.getFileExtension(url.getFileName()), dfltMime); } - public static String url2mime(final MultiProtocolURI url) { - return url == null ? "application/octet-stream" : ext2mime(MultiProtocolURI.getFileExtension(url.getFileName())); + public static String url2mime(final MultiProtocolURL url) { + return url == null ? "application/octet-stream" : ext2mime(MultiProtocolURL.getFileExtension(url.getFileName())); } } diff --git a/source/net/yacy/cora/document/ASCII.java b/source/net/yacy/cora/document/encoding/ASCII.java similarity index 99% rename from source/net/yacy/cora/document/ASCII.java rename to source/net/yacy/cora/document/encoding/ASCII.java index 578616fa6c..fe4d88b072 100644 --- a/source/net/yacy/cora/document/ASCII.java +++ b/source/net/yacy/cora/document/encoding/ASCII.java @@ -24,7 +24,7 @@ * If not, see . */ -package net.yacy.cora.document; +package net.yacy.cora.document.encoding; import java.util.Comparator; diff --git a/source/net/yacy/cora/document/UTF8.java b/source/net/yacy/cora/document/encoding/UTF8.java similarity index 99% rename from source/net/yacy/cora/document/UTF8.java rename to source/net/yacy/cora/document/encoding/UTF8.java index 4f098402f1..1d6de94a15 100644 --- a/source/net/yacy/cora/document/UTF8.java +++ b/source/net/yacy/cora/document/encoding/UTF8.java @@ -22,7 +22,7 @@ * If not, see . */ -package net.yacy.cora.document; +package net.yacy.cora.document.encoding; import java.io.UnsupportedEncodingException; import java.nio.charset.Charset; diff --git a/source/net/yacy/cora/document/Channel.java b/source/net/yacy/cora/document/feed/Channel.java similarity index 97% rename from source/net/yacy/cora/document/Channel.java rename to source/net/yacy/cora/document/feed/Channel.java index 337284af66..2200ae3ff5 100644 --- a/source/net/yacy/cora/document/Channel.java +++ b/source/net/yacy/cora/document/feed/Channel.java @@ -24,7 +24,8 @@ * If not, see . */ -package net.yacy.cora.document; +package net.yacy.cora.document.feed; + public interface Channel extends Iterable { diff --git a/source/net/yacy/cora/document/Channels.java b/source/net/yacy/cora/document/feed/Channels.java similarity index 96% rename from source/net/yacy/cora/document/Channels.java rename to source/net/yacy/cora/document/feed/Channels.java index f2cb6f58f1..d601486f1f 100644 --- a/source/net/yacy/cora/document/Channels.java +++ b/source/net/yacy/cora/document/feed/Channels.java @@ -24,7 +24,7 @@ * If not, see . */ -package net.yacy.cora.document; +package net.yacy.cora.document.feed; public class Channels { diff --git a/source/net/yacy/cora/document/Hit.java b/source/net/yacy/cora/document/feed/Hit.java similarity index 98% rename from source/net/yacy/cora/document/Hit.java rename to source/net/yacy/cora/document/feed/Hit.java index a8b5bc2b01..51a8afb238 100644 --- a/source/net/yacy/cora/document/Hit.java +++ b/source/net/yacy/cora/document/feed/Hit.java @@ -24,7 +24,7 @@ * If not, see . */ -package net.yacy.cora.document; +package net.yacy.cora.document.feed; import java.util.Date; import java.util.List; diff --git a/source/net/yacy/cora/document/RSSFeed.java b/source/net/yacy/cora/document/feed/RSSFeed.java similarity index 92% rename from source/net/yacy/cora/document/RSSFeed.java rename to source/net/yacy/cora/document/feed/RSSFeed.java index e39a661138..d0ab89f560 100644 --- a/source/net/yacy/cora/document/RSSFeed.java +++ b/source/net/yacy/cora/document/feed/RSSFeed.java @@ -18,7 +18,7 @@ * If not, see . */ -package net.yacy.cora.document; +package net.yacy.cora.document.feed; import java.net.MalformedURLException; import java.util.Collections; @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Set; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.util.ConcurrentLog; public class RSSFeed implements Iterable { @@ -53,11 +54,11 @@ public RSSFeed(final int maxsize) { * @param links * @param source */ - public RSSFeed(Set links, String source) { + public RSSFeed(Set links, String source) { this(Integer.MAX_VALUE); String u; RSSMessage message; - for (MultiProtocolURI uri: links) { + for (MultiProtocolURL uri: links) { u = uri.toNormalform(true); message = new RSSMessage(u, "", u); message.setAuthor(source); @@ -81,10 +82,10 @@ public String getImage() { return this.imageURL; } - public Set getLinks() { - Set links = new HashSet(); + public Set getLinks() { + Set links = new HashSet(); for (RSSMessage message: this.messages.values()) { - try {links.add(new MultiProtocolURI(message.getLink()));} catch (final MalformedURLException e) {} + try {links.add(new MultiProtocolURL(message.getLink()));} catch (final MalformedURLException e) {} } return links; } diff --git a/source/net/yacy/cora/document/RSSMessage.java b/source/net/yacy/cora/document/feed/RSSMessage.java similarity index 98% rename from source/net/yacy/cora/document/RSSMessage.java rename to source/net/yacy/cora/document/feed/RSSMessage.java index 1904d4ccc1..ee39fe545a 100644 --- a/source/net/yacy/cora/document/RSSMessage.java +++ b/source/net/yacy/cora/document/feed/RSSMessage.java @@ -22,7 +22,7 @@ * If not, see . */ -package net.yacy.cora.document; +package net.yacy.cora.document.feed; import java.text.ParseException; import java.util.ArrayList; @@ -37,6 +37,7 @@ import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.date.ISO8601Formatter; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.lod.vocabulary.DublinCore; import net.yacy.cora.lod.vocabulary.Geo; import net.yacy.cora.protocol.HeaderFramework; @@ -123,7 +124,7 @@ public RSSMessage(final String title, final String description, final String lin this.map.put(Token.guid.name(), artificialGuidPrefix + Integer.toHexString((title + description + link).hashCode())); } - public RSSMessage(final String title, final String description, final MultiProtocolURI link, final String guid) { + public RSSMessage(final String title, final String description, final MultiProtocolURL link, final String guid) { this.map = new HashMap(); if (title.length() > 0) this.map.put(Token.title.name(), title); if (description.length() > 0) this.map.put(Token.description.name(), description); diff --git a/source/net/yacy/cora/document/RSSReader.java b/source/net/yacy/cora/document/feed/RSSReader.java similarity index 98% rename from source/net/yacy/cora/document/RSSReader.java rename to source/net/yacy/cora/document/feed/RSSReader.java index b89a96a313..fdf2919b8e 100644 --- a/source/net/yacy/cora/document/RSSReader.java +++ b/source/net/yacy/cora/document/feed/RSSReader.java @@ -18,7 +18,7 @@ * If not, see . */ -package net.yacy.cora.document; +package net.yacy.cora.document.feed; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; @@ -30,7 +30,8 @@ import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; -import net.yacy.cora.document.RSSMessage.Token; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.feed.RSSMessage.Token; import org.xml.sax.Attributes; import org.xml.sax.EntityResolver; diff --git a/source/net/yacy/cora/document/id/AnchorURL.java b/source/net/yacy/cora/document/id/AnchorURL.java new file mode 100644 index 0000000000..d9c01065fb --- /dev/null +++ b/source/net/yacy/cora/document/id/AnchorURL.java @@ -0,0 +1,68 @@ +/** + * Anchor + * Copyright 2013 by Michael Peter Christen + * first published 15.09.2013 on http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + + +package net.yacy.cora.document.id; + +import java.net.MalformedURLException; +import java.util.Properties; + +public class AnchorURL extends DigestURL { + + private static final long serialVersionUID = 1586579902179962086L; + + private Properties properties; // may contain additional url properties, such as given in html a href-links + + public AnchorURL(final String url) throws MalformedURLException { + super(url); + this.properties = new Properties(); + } + + public AnchorURL(final MultiProtocolURL baseURL, final String relPath) throws MalformedURLException { + super(baseURL, relPath); + this.properties = new Properties(); + } + + public AnchorURL(final String protocol, final String host, final int port, final String path) throws MalformedURLException { + super(protocol, host, port, path); + this.properties = new Properties(); + } + + public Properties getProperties() { + return this.properties; + } + + public static AnchorURL newAnchor(final DigestURL baseURL, String relPath) throws MalformedURLException { + if (relPath.startsWith("//")) { + // patch for urls starting with "//" which can be found in the wild + relPath = (baseURL == null) ? "http:" + relPath : baseURL.getProtocol() + ":" + relPath; + } + if ((baseURL == null) || + isHTTP(relPath) || + isHTTPS(relPath) || + isFTP(relPath) || + isFile(relPath) || + isSMB(relPath)/*|| + relPath.contains(":") && patternMail.matcher(relPath.toLowerCase()).find()*/) { + return new AnchorURL(relPath); + } + return new AnchorURL(baseURL, relPath); + } +} diff --git a/source/net/yacy/kelondro/data/meta/DigestURI.java b/source/net/yacy/cora/document/id/DigestURL.java similarity index 77% rename from source/net/yacy/kelondro/data/meta/DigestURI.java rename to source/net/yacy/cora/document/id/DigestURL.java index debb0b7e36..48e9cee6c6 100644 --- a/source/net/yacy/kelondro/data/meta/DigestURI.java +++ b/source/net/yacy/cora/document/id/DigestURL.java @@ -1,28 +1,24 @@ -// DigestURI.java -// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany -// first published 13.07.2006 on http://yacy.net -// -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package net.yacy.kelondro.data.meta; +/** + * DigestURL + * Copyright 2006 by Michael Peter Christen + * first published 13.07.2006 on http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ + +package net.yacy.cora.document.id; // this class exist to provide a system-wide normal form representation of urls, // and to prevent that java.net.URL usage causes DNS queries which are used in java.net. @@ -31,18 +27,17 @@ import java.io.Serializable; import java.net.MalformedURLException; import java.util.HashSet; +import java.util.Properties; import java.util.Set; import java.util.regex.Pattern; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.Digest; import net.yacy.cora.protocol.Domains; import net.yacy.cora.util.ByteArray; import net.yacy.cora.util.CommonPattern; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.index.RowHandleSet; /** * URI-object providing YaCy-hash computation @@ -51,13 +46,14 @@ * For URIs pointing to resources not globally available, * the domainhash-part gets one reserved value */ -public class DigestURI extends MultiProtocolURI implements Serializable { +public class DigestURL extends MultiProtocolURL implements Serializable { private static final long serialVersionUID = -1173233022912141885L; public static final int TLD_any_zone_filter = 255; // from TLD zones can be filtered during search; this is the catch-all filter // class variables private byte[] hash; + private Properties properties; // may contain additional url properties, such as given in html a href-links /** * Shortcut, calculate hash for shorted url/hostname @@ -67,9 +63,9 @@ public class DigestURI extends MultiProtocolURI implements Serializable { public static String hosthash(final String host) { String h = host; if (!h.startsWith("http://")) h = "http://" + h; - DigestURI url = null; + DigestURL url = null; try { - url = new DigestURI(h); + url = new DigestURL(h); } catch (final MalformedURLException e) { ConcurrentLog.logException(e); return null; @@ -111,16 +107,17 @@ public static Set hosthashess(String hosthashes) { /** * DigestURI from File */ - public DigestURI(final File file) throws MalformedURLException { + public DigestURL(final File file) throws MalformedURLException { this("file", "", -1, file.getAbsolutePath()); } /** * DigestURI from URI string */ - public DigestURI(final String url) throws MalformedURLException { + public DigestURL(final String url) throws MalformedURLException { super(url); this.hash = null; + this.properties = new Properties(); } /** @@ -129,43 +126,36 @@ public DigestURI(final String url) throws MalformedURLException { * @param hash already calculated hash for url * @throws MalformedURLException */ - public DigestURI(final String url, final byte[] hash) throws MalformedURLException { + public DigestURL(final String url, final byte[] hash) throws MalformedURLException { super(url); this.hash = hash; + this.properties = new Properties(); } - - /** - * DigestURI from general URI - * @param u - */ - /* - private DigestURI(final MultiProtocolURI u) { - super(u); - this.hash = (u instanceof DigestURI) ? ((DigestURI) u).hash : null; - } - */ /** * DigestURI from general URI, hash already calculated * @param baseURL * @param hash */ - public DigestURI(final MultiProtocolURI baseURL, final byte[] hash) { + public DigestURL(final MultiProtocolURL baseURL, final byte[] hash) { super(baseURL); this.hash = hash; + this.properties = new Properties(); } - public DigestURI(final MultiProtocolURI baseURL, final String relPath) throws MalformedURLException { + public DigestURL(final MultiProtocolURL baseURL, final String relPath) throws MalformedURLException { super(baseURL, relPath); this.hash = null; + this.properties = new Properties(); } - public DigestURI(final String protocol, final String host, final int port, final String path) throws MalformedURLException { + public DigestURL(final String protocol, final String host, final int port, final String path) throws MalformedURLException { super(protocol, host, port, path); this.hash = null; + this.properties = new Properties(); } - public static DigestURI newURL(final DigestURI baseURL, String relPath) throws MalformedURLException { + public static DigestURL newURL(final DigestURL baseURL, String relPath) throws MalformedURLException { if (relPath.startsWith("//")) { // patch for urls starting with "//" which can be found in the wild relPath = (baseURL == null) ? "http:" + relPath : baseURL.getProtocol() + ":" + relPath; @@ -177,13 +167,17 @@ public static DigestURI newURL(final DigestURI baseURL, String relPath) throws M isFile(relPath) || isSMB(relPath)/*|| relPath.contains(":") && patternMail.matcher(relPath.toLowerCase()).find()*/) { - return new DigestURI(relPath); + return new DigestURL(relPath); } - return new DigestURI(baseURL, relPath); + return new DigestURL(baseURL, relPath); } private int hashCache = Integer.MIN_VALUE; // if this is used in a compare method many times, a cache is useful + public Properties getProperties() { + return this.properties; + } + @Override public int hashCode() { if (this.hashCache == Integer.MIN_VALUE) { @@ -302,25 +296,6 @@ private static char subdomPortPath(final String subdom, final int port, final St public final boolean probablyRootURL() { return this.path.length() <= 1 || rootPattern.matcher(this.path).matches(); } - - public RowHandleSet getPossibleRootHashes() { - RowHandleSet rootCandidates = new RowHandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 10); - String rootStub = this.getProtocol() + "://" + this.getHost(); - try { - rootCandidates.put(new DigestURI(rootStub).hash()); - rootCandidates.put(new DigestURI(rootStub + "/").hash()); - rootCandidates.put(new DigestURI(rootStub + "/index.htm").hash()); - rootCandidates.put(new DigestURI(rootStub + "/index.html").hash()); - rootCandidates.put(new DigestURI(rootStub + "/index.php").hash()); - rootCandidates.put(new DigestURI(rootStub + "/home.htm").hash()); - rootCandidates.put(new DigestURI(rootStub + "/home.html").hash()); - rootCandidates.put(new DigestURI(rootStub + "/home.php").hash()); - rootCandidates.put(new DigestURI(rootStub + "/default.htm").hash()); - rootCandidates.put(new DigestURI(rootStub + "/default.html").hash()); - rootCandidates.put(new DigestURI(rootStub + "/default.php").hash()); - } catch (final Throwable e) {} - return rootCandidates; - } private static final String hosthash5(final String protocol, final String host, final int port) { if (host == null) { diff --git a/source/net/yacy/cora/document/MultiProtocolURI.java b/source/net/yacy/cora/document/id/MultiProtocolURL.java similarity index 98% rename from source/net/yacy/cora/document/MultiProtocolURI.java rename to source/net/yacy/cora/document/id/MultiProtocolURL.java index 950f077f45..bff784868c 100644 --- a/source/net/yacy/cora/document/MultiProtocolURI.java +++ b/source/net/yacy/cora/document/id/MultiProtocolURL.java @@ -23,7 +23,7 @@ */ -package net.yacy.cora.document; +package net.yacy.cora.document.id; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; @@ -47,9 +47,10 @@ import jcifs.smb.SmbException; import jcifs.smb.SmbFile; import jcifs.smb.SmbFileInputStream; -import net.yacy.cora.document.Punycode.PunycodeException; import net.yacy.cora.document.analysis.Classification; import net.yacy.cora.document.analysis.Classification.ContentDomain; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.Punycode.PunycodeException; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.TimeoutRequest; @@ -61,9 +62,9 @@ * MultiProtocolURI provides a URL object for multiple protocols like http, https, ftp, smb and file * */ -public class MultiProtocolURI implements Serializable, Comparable { +public class MultiProtocolURL implements Serializable, Comparable { - public static final MultiProtocolURI POISON = new MultiProtocolURI(); // poison pill for concurrent link generators + public static final MultiProtocolURL POISON = new MultiProtocolURL(); // poison pill for concurrent link generators private static final Pattern ampPattern = Pattern.compile(Pattern.quote("&")); private static final long serialVersionUID = -1173233022912141884L; @@ -96,7 +97,7 @@ public static final void initSessionIDNames(final Set idNames) { /** * initialization of a MultiProtocolURI to produce poison pills for concurrent blocking queues */ - public MultiProtocolURI() { + public MultiProtocolURL() { this.protocol = null; this.host = null; this.hostAddress = null; @@ -108,11 +109,11 @@ public MultiProtocolURI() { this.port = -1; } - public MultiProtocolURI(final File file) throws MalformedURLException { + public MultiProtocolURL(final File file) throws MalformedURLException { this("file", "", -1, file.getAbsolutePath()); } - protected MultiProtocolURI(final MultiProtocolURI url) { + protected MultiProtocolURL(final MultiProtocolURL url) { this.protocol = url.protocol; this.host = url.host; this.hostAddress = null; @@ -124,7 +125,7 @@ protected MultiProtocolURI(final MultiProtocolURI url) { this.port = url.port; } - public MultiProtocolURI(String url) throws MalformedURLException { + public MultiProtocolURL(String url) throws MalformedURLException { if (url == null) throw new MalformedURLException("url string is null"); this.hostAddress = null; @@ -275,7 +276,7 @@ public final ContentDomain getContentDomain() { return this.contentDomain; } - public static MultiProtocolURI newURL(final String baseURL, String relPath) throws MalformedURLException { + public static MultiProtocolURL newURL(final String baseURL, String relPath) throws MalformedURLException { if (relPath.startsWith("//")) { // patch for urls starting with "//" which can be found in the wild relPath = "http:" + relPath; @@ -287,12 +288,12 @@ public static MultiProtocolURI newURL(final String baseURL, String relPath) thro isFile(relPath) || isSMB(relPath)/*|| relPath.contains(":") && patternMail.matcher(relPath.toLowerCase()).find()*/) { - return new MultiProtocolURI(relPath); + return new MultiProtocolURL(relPath); } - return new MultiProtocolURI(new MultiProtocolURI(baseURL), relPath); + return new MultiProtocolURL(new MultiProtocolURL(baseURL), relPath); } - public static MultiProtocolURI newURL(final MultiProtocolURI baseURL, String relPath) throws MalformedURLException { + public static MultiProtocolURL newURL(final MultiProtocolURL baseURL, String relPath) throws MalformedURLException { if (relPath.startsWith("//")) { // patch for urls starting with "//" which can be found in the wild relPath = (baseURL == null) ? "http:" + relPath : baseURL.getProtocol() + ":" + relPath; @@ -304,12 +305,12 @@ public static MultiProtocolURI newURL(final MultiProtocolURI baseURL, String rel isFile(relPath) || isSMB(relPath)/*|| relPath.contains(":") && patternMail.matcher(relPath.toLowerCase()).find()*/) { - return new MultiProtocolURI(relPath); + return new MultiProtocolURL(relPath); } - return new MultiProtocolURI(baseURL, relPath); + return new MultiProtocolURL(baseURL, relPath); } - public MultiProtocolURI(final MultiProtocolURI baseURL, String relPath) throws MalformedURLException { + public MultiProtocolURL(final MultiProtocolURL baseURL, String relPath) throws MalformedURLException { if (baseURL == null) throw new MalformedURLException("base URL is null"); if (relPath == null) throw new MalformedURLException("relPath is null"); @@ -361,7 +362,7 @@ public MultiProtocolURI(final MultiProtocolURI baseURL, String relPath) throws M escape(); } - public MultiProtocolURI(final String protocol, String host, final int port, final String path) throws MalformedURLException { + public MultiProtocolURL(final String protocol, String host, final int port, final String path) throws MalformedURLException { if (protocol == null) throw new MalformedURLException("protocol is null"); if (host.indexOf(':') >= 0 && host.charAt(0) != '[') host = '[' + host + ']'; // IPv6 host must be enclosed in square brackets this.protocol = protocol; @@ -948,8 +949,8 @@ public int hashCode() { public boolean equals(final Object obj) { if (this == obj) return true; if (obj == null) return false; - if (!(obj instanceof MultiProtocolURI)) return false; - final MultiProtocolURI other = (MultiProtocolURI) obj; + if (!(obj instanceof MultiProtocolURL)) return false; + final MultiProtocolURL other = (MultiProtocolURL) obj; return ((this.protocol == null && other.protocol == null) || (this.protocol != null && other.protocol != null && this.protocol.equals(other.protocol))) && @@ -961,7 +962,7 @@ public boolean equals(final Object obj) { } @Override - public int compareTo(final MultiProtocolURI h) { + public int compareTo(final MultiProtocolURL h) { int c; if (this.protocol != null && h.protocol != null && (c = this.protocol.compareTo(h.protocol)) != 0) return c; if (this.host != null && h.host != null && (c = this.host.compareTo(h.host)) != 0) return c; @@ -2167,12 +2168,12 @@ public static void main(final String[] args) { }; //MultiProtocolURI.initSessionIDNames(FileUtils.loadList(new File("defaults/sessionid.names"))); String environment, url; - MultiProtocolURI aURL, aURL1; + MultiProtocolURL aURL, aURL1; java.net.URL jURL; for (String[] element : test) { environment = element[0]; url = element[1]; - try {aURL = MultiProtocolURI.newURL(environment, url);} catch (final MalformedURLException e) {e.printStackTrace(); aURL = null;} + try {aURL = MultiProtocolURL.newURL(environment, url);} catch (final MalformedURLException e) {e.printStackTrace(); aURL = null;} if (environment == null) { try {jURL = new java.net.URL(url);} catch (final MalformedURLException e) {jURL = null;} } else { @@ -2190,7 +2191,7 @@ public static void main(final String[] args) { // check stability: the normalform of the normalform must be equal to the normalform if (aURL != null) try { - aURL1 = new MultiProtocolURI(aURL.toNormalform(false)); + aURL1 = new MultiProtocolURL(aURL.toNormalform(false)); if (!(aURL1.toNormalform(false).equals(aURL.toNormalform(false)))) { System.out.println("no stability for url:"); System.out.println("aURL0=" + aURL.toString()); diff --git a/source/net/yacy/cora/document/Punycode.java b/source/net/yacy/cora/document/id/Punycode.java similarity index 99% rename from source/net/yacy/cora/document/Punycode.java rename to source/net/yacy/cora/document/id/Punycode.java index 4f7887402b..cbbcbd63df 100644 --- a/source/net/yacy/cora/document/Punycode.java +++ b/source/net/yacy/cora/document/id/Punycode.java @@ -21,7 +21,7 @@ * USA */ -package net.yacy.cora.document; +package net.yacy.cora.document.id; public class Punycode { diff --git a/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java b/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java index 9721b0a355..f85645c825 100644 --- a/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java +++ b/source/net/yacy/cora/federate/opensearch/SRURSSConnector.java @@ -30,11 +30,11 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.RSSFeed; -import net.yacy.cora.document.RSSMessage; -import net.yacy.cora.document.RSSReader; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.feed.RSSFeed; +import net.yacy.cora.document.feed.RSSMessage; +import net.yacy.cora.document.feed.RSSReader; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.SearchAccumulator; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; @@ -163,9 +163,9 @@ public static RSSFeed loadSRURSS( final CacheStrategy cacheStrategy, final boolean global, final ClientIdentification.Agent agent) throws IOException { - MultiProtocolURI uri = null; + MultiProtocolURL uri = null; try { - uri = new MultiProtocolURI(rssSearchServiceURL); + uri = new MultiProtocolURL(rssSearchServiceURL); } catch (final MalformedURLException e) { throw new IOException("cora.Search failed asking peer '" + rssSearchServiceURL + "': bad url, " + e.getMessage()); } @@ -182,7 +182,7 @@ public static RSSFeed loadSRURSS( parts.put("nav", UTF8.StringBody("none")); // result = HTTPConnector.getConnector(userAgent == null ? MultiProtocolURI.yacybotUserAgent : userAgent).post(new MultiProtocolURI(rssSearchServiceURL), (int) timeout, uri.getHost(), parts); final HTTPClient httpClient = new HTTPClient(agent); - result = httpClient.POSTbytes(new MultiProtocolURI(rssSearchServiceURL), uri.getHost(), parts, false); + result = httpClient.POSTbytes(new MultiProtocolURL(rssSearchServiceURL), uri.getHost(), parts, false); final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result); if (reader == null) { diff --git a/source/net/yacy/cora/federate/solr/SchemaConfiguration.java b/source/net/yacy/cora/federate/solr/SchemaConfiguration.java index 1e0ab0f0d1..d976ae5152 100644 --- a/source/net/yacy/cora/federate/solr/SchemaConfiguration.java +++ b/source/net/yacy/cora/federate/solr/SchemaConfiguration.java @@ -34,11 +34,11 @@ import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrInputDocument; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.storage.Configuration; import net.yacy.cora.storage.HandleSet; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.index.Segment; import net.yacy.search.index.Segment.ReferenceReport; import net.yacy.search.index.Segment.ReferenceReportCache; @@ -78,7 +78,7 @@ public void fill(final SchemaConfiguration other, final boolean defaultActivated } } - public boolean postprocessing_doublecontent(Segment segment, Set uniqueURLs, SolrInputDocument sid, DigestURI url) { + public boolean postprocessing_doublecontent(Segment segment, Set uniqueURLs, SolrInputDocument sid, DigestURL url) { boolean changed = false; // FIND OUT IF THIS IS A DOUBLE DOCUMENT String hostid = url.hosthash(); @@ -149,7 +149,7 @@ public boolean postprocessing_doublecontent(Segment segment, Set uniqueU return changed; } - public boolean postprocessing_clickdepth(Segment segment, SolrDocument doc, SolrInputDocument sid, DigestURI url, SchemaDeclaration clickdepthfield) { + public boolean postprocessing_clickdepth(Segment segment, SolrDocument doc, SolrInputDocument sid, DigestURL url, SchemaDeclaration clickdepthfield) { if (!this.contains(clickdepthfield)) return false; // get new click depth and compare with old Integer oldclickdepth = (Integer) doc.getFieldValue(clickdepthfield.getSolrFieldName()); @@ -165,7 +165,7 @@ public boolean postprocessing_clickdepth(Segment segment, SolrDocument doc, Solr return false; } - public boolean postprocessing_references(ReferenceReportCache rrCache, SolrDocument doc, SolrInputDocument sid, DigestURI url, Map hostExtentCount) { + public boolean postprocessing_references(ReferenceReportCache rrCache, SolrDocument doc, SolrInputDocument sid, DigestURL url, Map hostExtentCount) { if (!(this.contains(CollectionSchema.references_i) || this.contains(CollectionSchema.references_internal_i) || this.contains(CollectionSchema.references_external_i) || this.contains(CollectionSchema.references_exthosts_i))) return false; diff --git a/source/net/yacy/cora/federate/solr/SolrServlet.java b/source/net/yacy/cora/federate/solr/SolrServlet.java index 6e3ebca81b..05506615fb 100644 --- a/source/net/yacy/cora/federate/solr/SolrServlet.java +++ b/source/net/yacy/cora/federate/solr/SolrServlet.java @@ -40,10 +40,9 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector; -import org.apache.lucene.document.Document; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.MultiMapSolrParams; import org.apache.solr.core.SolrCore; @@ -165,7 +164,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha int sz = ids.size(); for (int i = 0; i < sz; i++) { int id = iterator.nextDoc(); - Document doc = searcher.doc(id); + searcher.doc(id); } } } diff --git a/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java index 1795899e30..67b7a321bd 100644 --- a/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/AbstractSolrConnector.java @@ -33,7 +33,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.sorting.ClusteredScoreMap; import net.yacy.cora.sorting.ReversibleScoreMap; import net.yacy.cora.util.LookAheadIterator; diff --git a/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java index 89d0faa4d8..47fae1aabf 100644 --- a/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java +++ b/source/net/yacy/cora/federate/solr/connector/ConcurrentUpdateSolrConnector.java @@ -30,7 +30,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.sorting.ReversibleScoreMap; import net.yacy.cora.storage.HandleSet; import net.yacy.cora.util.ConcurrentLog; diff --git a/source/net/yacy/cora/federate/solr/connector/ShardSelection.java b/source/net/yacy/cora/federate/solr/connector/ShardSelection.java index 16aeb3f1ab..f317ab9d00 100644 --- a/source/net/yacy/cora/federate/solr/connector/ShardSelection.java +++ b/source/net/yacy/cora/federate/solr/connector/ShardSelection.java @@ -27,7 +27,7 @@ import java.security.NoSuchAlgorithmException; import java.util.concurrent.atomic.AtomicLong; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.util.ConcurrentLog; import net.yacy.search.schema.CollectionSchema; diff --git a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java index 3cbcc53b6d..f9fa62cef0 100644 --- a/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java +++ b/source/net/yacy/cora/federate/solr/instance/RemoteInstance.java @@ -28,7 +28,7 @@ import java.util.HashMap; import java.util.Map; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.Domains; import net.yacy.cora.util.ConcurrentLog; import net.yacy.search.schema.CollectionSchema; @@ -113,9 +113,9 @@ public RemoteInstance(final String url, final Collection coreNames, fina // Make a http client, connect using authentication. An url like // http://127.0.0.1:8983/solr/shard0 // is proper, and contains the core name as last element in the path - final MultiProtocolURI u; + final MultiProtocolURL u; try { - u = new MultiProtocolURI(this.solrurl + this.defaultCoreName); + u = new MultiProtocolURL(this.solrurl + this.defaultCoreName); } catch (final MalformedURLException e) { throw new IOException(e.getMessage()); } @@ -222,9 +222,9 @@ public SolrServer getServer(String name) { if (s != null) return s; // create new http server if (this.client != null) { - final MultiProtocolURI u; + final MultiProtocolURL u; try { - u = new MultiProtocolURI(this.solrurl + name); + u = new MultiProtocolURL(this.solrurl + name); } catch (final MalformedURLException e) { return null; } diff --git a/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java index 3a266d43be..27301eef5c 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/OpensearchResponseWriter.java @@ -31,8 +31,8 @@ import java.util.Map; import java.util.Set; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.RSSMessage; +import net.yacy.cora.document.feed.RSSMessage; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.lod.vocabulary.DublinCore; import net.yacy.cora.lod.vocabulary.Geo; import net.yacy.cora.lod.vocabulary.YaCyMetadata; @@ -181,7 +181,7 @@ public void write(final Writer writer, final SolrQueryRequest request, final Sol String u = value.stringValue(); solitaireTag(writer, RSSMessage.Token.link.name(), u); try { - MultiProtocolURI url = new MultiProtocolURI(u); + MultiProtocolURL url = new MultiProtocolURL(u); solitaireTag(writer, YaCyMetadata.host.getURIref(), url.getHost()); solitaireTag(writer, YaCyMetadata.path.getURIref(), url.getPath()); solitaireTag(writer, YaCyMetadata.file.getURIref(), url.getFileName()); diff --git a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java index edd1f2a66b..16e4eab901 100644 --- a/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java +++ b/source/net/yacy/cora/federate/solr/responsewriter/YJsonResponseWriter.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.Map; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.solr.responsewriter.OpensearchResponseWriter.ResHead; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.data.URLLicense; @@ -134,7 +134,7 @@ public void write(final Writer writer, final SolrQueryRequest request, final Sol List fields = doc.getFields(); int fieldc = fields.size(); List texts = new ArrayList(); - MultiProtocolURI url = null; + MultiProtocolURL url = null; String urlhash = null; List descriptions = new ArrayList(); String title = ""; @@ -153,12 +153,12 @@ public void write(final Writer writer, final SolrQueryRequest request, final Sol if (CollectionSchema.sku.getSolrFieldName().equals(fieldName)) { String u = value.stringValue(); try { - url = new MultiProtocolURI(u); + url = new MultiProtocolURL(u); String filename = url.getFileName(); solitaireTag(writer, "link", u); solitaireTag(writer, "file", filename); // get image license - if (MultiProtocolURI.isImage(filename)) URLLicense.aquireLicense(urlhash, url.toNormalform(true)); + if (MultiProtocolURL.isImage(filename)) URLLicense.aquireLicense(urlhash, url.toNormalform(true)); } catch (final MalformedURLException e) {} continue; } diff --git a/source/net/yacy/cora/federate/yacy/Distribution.java b/source/net/yacy/cora/federate/yacy/Distribution.java index b472bb2ee5..9382dd1336 100644 --- a/source/net/yacy/cora/federate/yacy/Distribution.java +++ b/source/net/yacy/cora/federate/yacy/Distribution.java @@ -20,8 +20,8 @@ package net.yacy.cora.federate.yacy; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; /** diff --git a/source/net/yacy/cora/federate/yacy/Peer.java b/source/net/yacy/cora/federate/yacy/Peer.java index 6b359c65ab..ee65036854 100644 --- a/source/net/yacy/cora/federate/yacy/Peer.java +++ b/source/net/yacy/cora/federate/yacy/Peer.java @@ -23,7 +23,7 @@ import java.io.Serializable; import java.util.HashMap; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.order.Base64Order; /** diff --git a/source/net/yacy/cora/federate/yacy/Peers.java b/source/net/yacy/cora/federate/yacy/Peers.java index 238dd88921..a3faeb6804 100644 --- a/source/net/yacy/cora/federate/yacy/Peers.java +++ b/source/net/yacy/cora/federate/yacy/Peers.java @@ -29,7 +29,7 @@ import java.util.Set; import java.util.TreeMap; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.federate.yacy.api.Network; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.http.HTTPClient; diff --git a/source/net/yacy/cora/geo/GeoLocation.java b/source/net/yacy/cora/geo/GeoLocation.java index 45cb68be5c..2d180ed462 100644 --- a/source/net/yacy/cora/geo/GeoLocation.java +++ b/source/net/yacy/cora/geo/GeoLocation.java @@ -24,7 +24,7 @@ import java.util.Comparator; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; public class GeoLocation extends IntegerGeoPoint implements Comparable, Comparator { diff --git a/source/net/yacy/cora/lod/JenaTripleStore.java b/source/net/yacy/cora/lod/JenaTripleStore.java index d7ecdad2c8..a4f93b60f7 100644 --- a/source/net/yacy/cora/lod/JenaTripleStore.java +++ b/source/net/yacy/cora/lod/JenaTripleStore.java @@ -14,7 +14,7 @@ import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.lod.vocabulary.DCTerms; import net.yacy.cora.lod.vocabulary.Geo; import net.yacy.cora.lod.vocabulary.HttpHeader; diff --git a/source/net/yacy/cora/lod/Literal.java b/source/net/yacy/cora/lod/Literal.java index c5794c5c27..94b0bea70b 100644 --- a/source/net/yacy/cora/lod/Literal.java +++ b/source/net/yacy/cora/lod/Literal.java @@ -26,7 +26,7 @@ import java.util.regex.Pattern; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.MultiProtocolURL; /** * A literal is the possible value for a predicate. @@ -55,7 +55,7 @@ public interface Literal { * assigned. * @return an url to a knowledge authority for the literal */ - public MultiProtocolURI getSubject(); + public MultiProtocolURL getSubject(); /** * if a resource is poorly annotated with metadata an it shall diff --git a/source/net/yacy/cora/lod/Node.java b/source/net/yacy/cora/lod/Node.java index d1b0fc34a6..c2f279b708 100644 --- a/source/net/yacy/cora/lod/Node.java +++ b/source/net/yacy/cora/lod/Node.java @@ -27,7 +27,7 @@ import java.util.HashMap; import java.util.Map; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.lod.vocabulary.Rdf; /** diff --git a/source/net/yacy/cora/lod/vocabulary/CreativeCommons.java b/source/net/yacy/cora/lod/vocabulary/CreativeCommons.java index 97950651ac..d21e11bf22 100644 --- a/source/net/yacy/cora/lod/vocabulary/CreativeCommons.java +++ b/source/net/yacy/cora/lod/vocabulary/CreativeCommons.java @@ -29,7 +29,7 @@ import java.util.Set; import java.util.regex.Pattern; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.lod.Literal; import net.yacy.cora.lod.Vocabulary; @@ -74,7 +74,7 @@ enum PermitLiteral implements Literal { Sharing("Sharing", "http://creativecommons.org/ns#Permission", ".*"); String terminal; - MultiProtocolURI subject; + MultiProtocolURL subject; Pattern discoveryPattern; private PermitLiteral( @@ -83,7 +83,7 @@ private PermitLiteral( String discoveryPattern) { this.terminal = terminal; try { - this.subject = subject == null ? null : new MultiProtocolURI(subject); + this.subject = subject == null ? null : new MultiProtocolURL(subject); } catch (final MalformedURLException e) { this.subject = null; } @@ -94,7 +94,7 @@ private PermitLiteral( public String getTerminal() { return this.terminal; } @Override - public MultiProtocolURI getSubject() { return this.subject; } + public MultiProtocolURL getSubject() { return this.subject; } @Override public Pattern getDiscoveryPattern() { return this.discoveryPattern; } @@ -110,7 +110,7 @@ enum RequirementLiteral implements Literal { LesserCopyleft("Lesser Copyleft", "http://creativecommons.org/ns#Requirement", ".*"); String terminal; - MultiProtocolURI subject; + MultiProtocolURL subject; Pattern discoveryPattern; private RequirementLiteral( @@ -119,7 +119,7 @@ private RequirementLiteral( String discoveryPattern) { this.terminal = terminal; try { - this.subject = subject == null ? null : new MultiProtocolURI(subject); + this.subject = subject == null ? null : new MultiProtocolURL(subject); } catch (final MalformedURLException e) { this.subject = null; } @@ -130,7 +130,7 @@ private RequirementLiteral( public String getTerminal() { return this.terminal; } @Override - public MultiProtocolURI getSubject() { return this.subject; } + public MultiProtocolURL getSubject() { return this.subject; } @Override public Pattern getDiscoveryPattern() { return this.discoveryPattern; } @@ -142,7 +142,7 @@ enum ProhibitionLiteral implements Literal { HighIncomeNationUse("High Income Nation Use", "http://creativecommons.org/ns#Prohibition", ".*"); String terminal; - MultiProtocolURI subject; + MultiProtocolURL subject; Pattern discoveryPattern; private ProhibitionLiteral( @@ -151,7 +151,7 @@ private ProhibitionLiteral( String discoveryPattern) { this.terminal = terminal; try { - this.subject = subject == null ? null : new MultiProtocolURI(subject); + this.subject = subject == null ? null : new MultiProtocolURL(subject); } catch (final MalformedURLException e) { this.subject = null; } @@ -162,7 +162,7 @@ private ProhibitionLiteral( public String getTerminal() { return this.terminal; } @Override - public MultiProtocolURI getSubject() { return this.subject; } + public MultiProtocolURL getSubject() { return this.subject; } @Override public Pattern getDiscoveryPattern() { return this.discoveryPattern; } diff --git a/source/net/yacy/cora/lod/vocabulary/YaCyMetadata.java b/source/net/yacy/cora/lod/vocabulary/YaCyMetadata.java index ee7662761b..2b8ecde4d9 100644 --- a/source/net/yacy/cora/lod/vocabulary/YaCyMetadata.java +++ b/source/net/yacy/cora/lod/vocabulary/YaCyMetadata.java @@ -27,7 +27,7 @@ import java.util.Set; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.lod.Literal; import net.yacy.cora.lod.Vocabulary; diff --git a/source/net/yacy/cora/order/Base64Order.java b/source/net/yacy/cora/order/Base64Order.java index 2a0d6dab8c..64d5eeec40 100644 --- a/source/net/yacy/cora/order/Base64Order.java +++ b/source/net/yacy/cora/order/Base64Order.java @@ -23,7 +23,7 @@ import java.io.Serializable; import java.util.Comparator; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; //ATTENTION! THIS CLASS SHALL NOT IMPORT FROM OTHER PACKAGES THAN CORA AND JRE //BECAUSE OTHERWISE THE DEBIAN INSTALLER FAILS! diff --git a/source/net/yacy/cora/order/Digest.java b/source/net/yacy/cora/order/Digest.java index 96f489f3fd..619beaab8a 100644 --- a/source/net/yacy/cora/order/Digest.java +++ b/source/net/yacy/cora/order/Digest.java @@ -38,7 +38,7 @@ import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.LinkedBlockingQueue; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.storage.ARC; import net.yacy.cora.storage.ConcurrentARC; import net.yacy.cora.util.Memory; diff --git a/source/net/yacy/cora/order/StringOrder.java b/source/net/yacy/cora/order/StringOrder.java index 56a0493b77..56f230e806 100644 --- a/source/net/yacy/cora/order/StringOrder.java +++ b/source/net/yacy/cora/order/StringOrder.java @@ -23,7 +23,7 @@ import java.io.Serializable; import java.util.Comparator; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; public class StringOrder implements Comparator, Serializable { diff --git a/source/net/yacy/cora/protocol/HeaderFramework.java b/source/net/yacy/cora/protocol/HeaderFramework.java index c7352e3997..43fb259a3b 100644 --- a/source/net/yacy/cora/protocol/HeaderFramework.java +++ b/source/net/yacy/cora/protocol/HeaderFramework.java @@ -39,11 +39,11 @@ import java.util.Vector; import java.util.concurrent.ConcurrentHashMap; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.CommonPattern; import net.yacy.cora.util.NumberTools; -import net.yacy.kelondro.data.meta.DigestURI; /** @@ -568,7 +568,7 @@ else if (httpVersion.equals("HTTP/1.1") && HeaderFramework.http1_1.containsKey(I theHeader.append("\r\n"); } - public static DigestURI getRequestURL(final HashMap conProp) throws MalformedURLException { + public static DigestURL getRequestURL(final HashMap conProp) throws MalformedURLException { String host = (String) conProp.get(HeaderFramework.CONNECTION_PROP_HOST); final String path = (String) conProp.get(HeaderFramework.CONNECTION_PROP_PATH); // always starts with leading '/' final String args = (String) conProp.get(HeaderFramework.CONNECTION_PROP_ARGS); // may be null if no args were given @@ -582,7 +582,7 @@ public static DigestURI getRequestURL(final HashMap conProp) thr host = host.substring(0, pos); } - final DigestURI url = new DigestURI("http", host, port, (args == null) ? path : path + "?" + args); + final DigestURL url = new DigestURL("http", host, port, (args == null) ? path : path + "?" + args); return url; } diff --git a/source/net/yacy/cora/protocol/RequestHeader.java b/source/net/yacy/cora/protocol/RequestHeader.java index faee711cc7..711ff9bfda 100644 --- a/source/net/yacy/cora/protocol/RequestHeader.java +++ b/source/net/yacy/cora/protocol/RequestHeader.java @@ -24,8 +24,8 @@ import java.util.Date; import java.util.Map; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; public class RequestHeader extends HeaderFramework { @@ -71,18 +71,18 @@ public RequestHeader(final Map reverseMappingCache, final Map> scancacheEntries() { * @param url * @return true if the url shall be part of a search result */ - public static boolean acceptURL(final MultiProtocolURI url) { + public static boolean acceptURL(final MultiProtocolURL url) { // if the scan range is empty, then all urls are accepted if (scancache == null || scancache.isEmpty()) return true; diff --git a/source/net/yacy/cora/protocol/ftp/FTPClient.java b/source/net/yacy/cora/protocol/ftp/FTPClient.java index 89baa1efc4..8cbbdd2111 100644 --- a/source/net/yacy/cora/protocol/ftp/FTPClient.java +++ b/source/net/yacy/cora/protocol/ftp/FTPClient.java @@ -65,7 +65,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.Domains; import net.yacy.cora.util.ConcurrentLog; diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index 7d401e4cdc..71b2591a42 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -43,8 +43,8 @@ import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.cora.protocol.Domains; @@ -318,7 +318,7 @@ public byte[] GETbytes(final String uri) throws IOException { * @return content bytes * @throws IOException */ - public byte[] GETbytes(final MultiProtocolURI url) throws IOException { + public byte[] GETbytes(final MultiProtocolURL url) throws IOException { return GETbytes(url, Integer.MAX_VALUE); } @@ -331,7 +331,7 @@ public byte[] GETbytes(final MultiProtocolURI url) throws IOException { * @throws IOException */ public byte[] GETbytes(final String uri, final int maxBytes) throws IOException { - return GETbytes(new MultiProtocolURI(uri), maxBytes); + return GETbytes(new MultiProtocolURL(uri), maxBytes); } @@ -343,7 +343,7 @@ public byte[] GETbytes(final String uri, final int maxBytes) throws IOException * @return content bytes * @throws IOException */ - public byte[] GETbytes(final MultiProtocolURI url, final int maxBytes) throws IOException { + public byte[] GETbytes(final MultiProtocolURL url, final int maxBytes) throws IOException { final boolean localhost = Domains.isLocalhost(url.getHost()); final String urix = url.toNormalform(true); HttpGet httpGet = null; @@ -367,7 +367,7 @@ public byte[] GETbytes(final MultiProtocolURI url, final int maxBytes) throws IO */ public void GET(final String uri) throws IOException { if (this.currentRequest != null) throw new IOException("Client is in use!"); - final MultiProtocolURI url = new MultiProtocolURI(uri); + final MultiProtocolURL url = new MultiProtocolURL(uri); final String urix = url.toNormalform(true); HttpGet httpGet = null; try { @@ -389,7 +389,7 @@ public void GET(final String uri) throws IOException { * @throws IOException */ public HttpResponse HEADResponse(final String uri) throws IOException { - final MultiProtocolURI url = new MultiProtocolURI(uri); + final MultiProtocolURL url = new MultiProtocolURL(uri); final HttpHead httpHead = new HttpHead(url.toNormalform(true)); httpHead.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state setHost(url.getHost()); // overwrite resolved IP, needed for shared web hosting DO NOT REMOVE, see http://en.wikipedia.org/wiki/Shared_web_hosting_service @@ -411,7 +411,7 @@ public HttpResponse HEADResponse(final String uri) throws IOException { */ public void POST(final String uri, final InputStream instream, final long length) throws IOException { if (this.currentRequest != null) throw new IOException("Client is in use!"); - final MultiProtocolURI url = new MultiProtocolURI(uri); + final MultiProtocolURL url = new MultiProtocolURL(uri); final HttpPost httpPost = new HttpPost(url.toNormalform(true)); httpPost.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state String host = url.getHost(); @@ -434,7 +434,7 @@ public void POST(final String uri, final InputStream instream, final long length * @throws IOException */ public byte[] POSTbytes(final String uri, final Map parts, final boolean usegzip) throws IOException { - final MultiProtocolURI url = new MultiProtocolURI(uri); + final MultiProtocolURL url = new MultiProtocolURL(uri); return POSTbytes(url, url.getHost(), parts, usegzip); } @@ -448,7 +448,7 @@ public byte[] POSTbytes(final String uri, final Map parts, * @return response body * @throws IOException */ - public byte[] POSTbytes(final MultiProtocolURI url, final String vhost, final Map post, final boolean usegzip) throws IOException { + public byte[] POSTbytes(final MultiProtocolURL url, final String vhost, final Map post, final boolean usegzip) throws IOException { final HttpPost httpPost = new HttpPost(url.toNormalform(true)); httpPost.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state @@ -480,7 +480,7 @@ public byte[] POSTbytes(final MultiProtocolURI url, final String vhost, final Ma * @throws IOException */ public byte[] POSTbytes(final String uri, final InputStream instream, final long length) throws IOException { - final MultiProtocolURI url = new MultiProtocolURI(uri); + final MultiProtocolURL url = new MultiProtocolURL(uri); final HttpPost httpPost = new HttpPost(url.toNormalform(true)); httpPost.addHeader(new BasicHeader("Connection", "close")); // don't keep alive, prevent CLOSE_WAIT state String host = url.getHost(); diff --git a/source/net/yacy/cora/protocol/http/LinkExtractor.java b/source/net/yacy/cora/protocol/http/LinkExtractor.java index 59b13c0166..7935492864 100644 --- a/source/net/yacy/cora/protocol/http/LinkExtractor.java +++ b/source/net/yacy/cora/protocol/http/LinkExtractor.java @@ -28,18 +28,18 @@ import java.util.WeakHashMap; import java.util.regex.Pattern; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.MultiProtocolURL; public class LinkExtractor { private static final char lb = '<', rb = '>', dquotes = '"', space = ' '; private static final Object PRESENT = new Object(); - private WeakHashMap links; + private WeakHashMap links; private Pattern blackpattern; public LinkExtractor(Pattern blackpattern) { - this.links = new WeakHashMap(); + this.links = new WeakHashMap(); this.blackpattern = blackpattern; } @@ -55,7 +55,7 @@ public void scrape(String text) { if (u.endsWith(".")) u = u.substring(0, u.length() - 1); // remove the '.' that was appended above s = p + 1; if (this.blackpattern.matcher(u).matches()) continue; - try {links.put(new MultiProtocolURI(u), PRESENT);} catch (final MalformedURLException e) {} + try {links.put(new MultiProtocolURL(u), PRESENT);} catch (final MalformedURLException e) {} } } @@ -63,10 +63,10 @@ public void scrape(String text) { * return the links in the text in the order as they appear * @return a list of urls */ - public MultiProtocolURI[] getLinks() { - MultiProtocolURI[] urls = new MultiProtocolURI[this.links.size()]; + public MultiProtocolURL[] getLinks() { + MultiProtocolURL[] urls = new MultiProtocolURL[this.links.size()]; int i = 0; - for (MultiProtocolURI uri: this.links.keySet()) urls[i++] = uri; + for (MultiProtocolURL uri: this.links.keySet()) urls[i++] = uri; return urls; } diff --git a/source/net/yacy/cora/storage/AbstractMapStore.java b/source/net/yacy/cora/storage/AbstractMapStore.java index df030c3e2a..899a472add 100644 --- a/source/net/yacy/cora/storage/AbstractMapStore.java +++ b/source/net/yacy/cora/storage/AbstractMapStore.java @@ -30,7 +30,7 @@ import java.util.Map; import java.util.Set; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; public abstract class AbstractMapStore implements MapStore { diff --git a/source/net/yacy/cora/storage/KeyList.java b/source/net/yacy/cora/storage/KeyList.java index c259e4e0b0..37cccec6ff 100644 --- a/source/net/yacy/cora/storage/KeyList.java +++ b/source/net/yacy/cora/storage/KeyList.java @@ -36,7 +36,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.zip.GZIPInputStream; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; /** * a key list is a file which contains a list of key words; each line one word diff --git a/source/net/yacy/cora/util/ByteArray.java b/source/net/yacy/cora/util/ByteArray.java index 5ea2ca74a7..e117e96702 100644 --- a/source/net/yacy/cora/util/ByteArray.java +++ b/source/net/yacy/cora/util/ByteArray.java @@ -22,7 +22,7 @@ import java.util.HashMap; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; /** diff --git a/source/net/yacy/cora/util/ByteBuffer.java b/source/net/yacy/cora/util/ByteBuffer.java index 8ff26bf677..208d18103a 100644 --- a/source/net/yacy/cora/util/ByteBuffer.java +++ b/source/net/yacy/cora/util/ByteBuffer.java @@ -27,7 +27,7 @@ import java.util.Iterator; import java.util.List; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; public final class ByteBuffer extends OutputStream { diff --git a/source/net/yacy/cora/document/JSONArray.java b/source/net/yacy/cora/util/JSONArray.java similarity index 99% rename from source/net/yacy/cora/document/JSONArray.java rename to source/net/yacy/cora/util/JSONArray.java index 7590b1b952..9bbfc1779a 100644 --- a/source/net/yacy/cora/document/JSONArray.java +++ b/source/net/yacy/cora/util/JSONArray.java @@ -22,7 +22,7 @@ of this software and associated documentation files (the "Software"), to deal SOFTWARE. */ -package net.yacy.cora.document; +package net.yacy.cora.util; import java.io.IOException; import java.io.Writer; diff --git a/source/net/yacy/cora/document/JSONException.java b/source/net/yacy/cora/util/JSONException.java similarity index 91% rename from source/net/yacy/cora/document/JSONException.java rename to source/net/yacy/cora/util/JSONException.java index 0aed962da1..56cb233465 100644 --- a/source/net/yacy/cora/document/JSONException.java +++ b/source/net/yacy/cora/util/JSONException.java @@ -4,7 +4,7 @@ * @version 2008-09-18 */ -package net.yacy.cora.document; +package net.yacy.cora.util; public class JSONException extends Exception { /** diff --git a/source/net/yacy/cora/document/JSONObject.java b/source/net/yacy/cora/util/JSONObject.java similarity index 99% rename from source/net/yacy/cora/document/JSONObject.java rename to source/net/yacy/cora/util/JSONObject.java index 8afbbfdc40..c3ef229e08 100644 --- a/source/net/yacy/cora/document/JSONObject.java +++ b/source/net/yacy/cora/util/JSONObject.java @@ -22,7 +22,7 @@ of this software and associated documentation files (the "Software"), to deal SOFTWARE. */ -package net.yacy.cora.document; +package net.yacy.cora.util; import java.io.IOException; import java.io.Writer; diff --git a/source/net/yacy/cora/document/JSONTokener.java b/source/net/yacy/cora/util/JSONTokener.java similarity index 99% rename from source/net/yacy/cora/document/JSONTokener.java rename to source/net/yacy/cora/util/JSONTokener.java index e35f741112..2d68be0929 100644 --- a/source/net/yacy/cora/document/JSONTokener.java +++ b/source/net/yacy/cora/util/JSONTokener.java @@ -30,7 +30,7 @@ of this software and associated documentation files (the "Software"), to deal * @version 2010-02-02 */ -package net.yacy.cora.document; +package net.yacy.cora.util; import java.io.BufferedReader; import java.io.IOException; diff --git a/source/net/yacy/crawler/Balancer.java b/source/net/yacy/crawler/Balancer.java index f00510ff81..67cbabeb55 100644 --- a/source/net/yacy/crawler/Balancer.java +++ b/source/net/yacy/crawler/Balancer.java @@ -40,8 +40,9 @@ import org.openjena.atlas.logging.Log; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.ClientIdentification; @@ -55,7 +56,6 @@ import net.yacy.crawler.data.Latency; import net.yacy.crawler.retrieval.Request; import net.yacy.crawler.robots.RobotsTxt; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.index.BufferedObjectIndex; import net.yacy.kelondro.index.Row; @@ -301,7 +301,7 @@ public Map getDomainStackHosts(RobotsTxt robots) { * @param crawlURL * @return the sleep time in milliseconds; may be negative for no sleep time */ - private long getDomainSleepTime(final RobotsTxt robots, final CrawlProfile profileEntry, final DigestURI crawlURL) { + private long getDomainSleepTime(final RobotsTxt robots, final CrawlProfile profileEntry, final DigestURL crawlURL) { if (profileEntry == null) return 0; long sleeptime = ( profileEntry.cacheStrategy() == CacheStrategy.CACHEONLY || @@ -319,7 +319,7 @@ private long getDomainSleepTime(final RobotsTxt robots, final CrawlProfile profi * @param crawlURL * @return */ - private long getRobotsTime(final RobotsTxt robots, final DigestURI crawlURL, ClientIdentification.Agent agent) { + private long getRobotsTime(final RobotsTxt robots, final DigestURL crawlURL, ClientIdentification.Agent agent) { long sleeptime = Latency.waitingRobots(crawlURL, robots, agent); // this uses the robots.txt database and may cause a loading of robots.txt from the server return sleeptime < 0 ? 0 : sleeptime; } diff --git a/source/net/yacy/crawler/CrawlQueue.java b/source/net/yacy/crawler/CrawlQueue.java index 3c67774790..ff6a4637e8 100644 --- a/source/net/yacy/crawler/CrawlQueue.java +++ b/source/net/yacy/crawler/CrawlQueue.java @@ -24,8 +24,9 @@ import java.io.IOException; import java.util.Iterator; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.ClientIdentification; @@ -37,7 +38,6 @@ import net.yacy.crawler.data.Latency; import net.yacy.crawler.retrieval.Request; import net.yacy.crawler.robots.RobotsTxt; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.index.BufferedObjectIndex; import net.yacy.kelondro.index.Row; @@ -210,7 +210,7 @@ public String push(final Request entry, CrawlProfile profile, final RobotsTxt ro * @param crawlURL * @return the sleep time in milliseconds; may be negative for no sleep time */ - private long getDomainSleepTime(final RobotsTxt robots, final CrawlProfile profileEntry, final DigestURI crawlURL) { + private long getDomainSleepTime(final RobotsTxt robots, final CrawlProfile profileEntry, final DigestURL crawlURL) { if (profileEntry == null) return 0; long sleeptime = ( profileEntry.cacheStrategy() == CacheStrategy.CACHEONLY || @@ -228,7 +228,7 @@ private long getDomainSleepTime(final RobotsTxt robots, final CrawlProfile profi * @param crawlURL * @return */ - private long getRobotsTime(final RobotsTxt robots, final DigestURI crawlURL, ClientIdentification.Agent agent) { + private long getRobotsTime(final RobotsTxt robots, final DigestURL crawlURL, ClientIdentification.Agent agent) { long sleeptime = Latency.waitingRobots(crawlURL, robots, agent); // this uses the robots.txt database and may cause a loading of robots.txt from the server return sleeptime < 0 ? 0 : sleeptime; } diff --git a/source/net/yacy/crawler/CrawlStacker.java b/source/net/yacy/crawler/CrawlStacker.java index df0f49d34e..d21b487dd2 100644 --- a/source/net/yacy/crawler/CrawlStacker.java +++ b/source/net/yacy/crawler/CrawlStacker.java @@ -31,16 +31,16 @@ import java.util.Date; import java.util.List; import java.util.Locale; -import java.util.Map; -import java.util.Properties; import java.util.concurrent.BlockingQueue; import java.util.concurrent.atomic.AtomicInteger; import net.yacy.contentcontrol.ContentControlFilterUpdateThread; -import net.yacy.cora.document.ASCII; import net.yacy.cora.document.analysis.Classification.ContentDomain; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.AnchorURL; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.ftp.FTPClient; @@ -58,7 +58,6 @@ import net.yacy.crawler.retrieval.SMBLoader; import net.yacy.crawler.robots.RobotsTxt; import net.yacy.kelondro.data.citation.CitationReference; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.rwi.IndexCell; import net.yacy.kelondro.workflow.WorkflowProcessor; import net.yacy.peers.SeedDB; @@ -167,7 +166,7 @@ public void enqueueEntry(final Request entry) { if (this.log.isFinest()) this.log.finest("ENQUEUE " + entry.url() + ", referer=" + entry.referrerhash() + ", initiator=" + ((entry.initiator() == null) ? "" : ASCII.String(entry.initiator())) + ", name=" + entry.name() + ", appdate=" + entry.appdate() + ", depth=" + entry.depth()); this.requestQueue.enQueue(entry); } - public void enqueueEntriesAsynchronous(final byte[] initiator, final String profileHandle, final Map hyperlinks) { + public void enqueueEntriesAsynchronous(final byte[] initiator, final String profileHandle, final List hyperlinks) { new Thread() { @Override public void run() { @@ -177,12 +176,11 @@ public void run() { }.start(); } - private void enqueueEntries(final byte[] initiator, final String profileHandle, final Map hyperlinks, final boolean replace) { - for (final Map.Entry e: hyperlinks.entrySet()) { - if (e.getKey() == null) continue; + private void enqueueEntries(final byte[] initiator, final String profileHandle, final List hyperlinks, final boolean replace) { + for (final DigestURL url: hyperlinks) { + if (url == null) continue; // delete old entry, if exists to force a re-load of the url (thats wanted here) - final DigestURI url = e.getKey(); final byte[] urlhash = url.hash(); if (replace) { this.indexSegment.fulltext().remove(urlhash); @@ -197,7 +195,7 @@ private void enqueueEntries(final byte[] initiator, final String profileHandle, u = u + "/index.html"; } try { - final byte[] uh = new DigestURI(u).hash(); + final byte[] uh = new DigestURL(u).hash(); this.indexSegment.fulltext().remove(uh); this.nextQueue.noticeURL.removeByURLHash(uh); this.nextQueue.errorURL.remove(uh); @@ -213,7 +211,7 @@ private void enqueueEntries(final byte[] initiator, final String profileHandle, initiator, url, null, - e.getValue().getProperty("name", ""), + url.getProperties().getProperty("name", ""), new Date(), profileHandle, 0, @@ -238,9 +236,9 @@ public void run() { while ((entry = queue.take()) != FTPClient.POISON_entryInfo) { // delete old entry, if exists to force a re-load of the url (thats wanted here) - DigestURI url = null; + DigestURL url = null; try { - url = new DigestURI("ftp://" + host + (port == 21 ? "" : ":" + port) + MultiProtocolURI.escape(entry.name)); + url = new DigestURL("ftp://" + host + (port == 21 ? "" : ":" + port) + MultiProtocolURL.escape(entry.name)); } catch (final MalformedURLException e) { continue; } @@ -256,7 +254,7 @@ public void run() { initiator, url, null, - MultiProtocolURI.unescape(entry.name), + MultiProtocolURL.unescape(entry.name), entry.date, profileHandle, 0, @@ -277,7 +275,7 @@ public void run() { * @param url * @return null if successfull, a reason string if not successful */ - public String stackSimpleCrawl(final DigestURI url) { + public String stackSimpleCrawl(final DigestURL url) { final CrawlProfile pe = this.crawler.defaultSurrogateProfile; return stackCrawl(new Request( this.peers.mySeed().hash.getBytes(), @@ -373,7 +371,7 @@ public String stackCrawl(final Request entry) { return null; } - public String checkAcceptance(final DigestURI url, final CrawlProfile profile, final int depth) { + public String checkAcceptance(final DigestURL url, final CrawlProfile profile, final int depth) { // check if the protocol is supported final String urlProtocol = url.getProtocol(); @@ -512,7 +510,7 @@ public String checkAcceptance(final DigestURI url, final CrawlProfile profile, f * @param url * @return null if the url can be accepted, a string containing a rejection reason if the url cannot be accepted */ - public String urlInAcceptedDomain(final DigestURI url) { + public String urlInAcceptedDomain(final DigestURL url) { // returns true if the url can be accepted according to network.unit.domain if (url == null) return "url is null"; // check domainList from network-definition @@ -560,7 +558,7 @@ public String urlInAcceptedDomainHash(final byte[] urlhash) { // returns true if the url can be accepted according to network.unit.domain if (urlhash == null) return "url is null"; // check if this is a local address and we are allowed to index local pages: - final boolean local = DigestURI.isLocal(urlhash); + final boolean local = DigestURL.isLocal(urlhash); if (this.acceptLocalURLs && local) return null; if (this.acceptGlobalURLs && !local) return null; return (local) ? diff --git a/source/net/yacy/crawler/CrawlSwitchboard.java b/source/net/yacy/crawler/CrawlSwitchboard.java index f721d49bea..d2ecf04373 100644 --- a/source/net/yacy/crawler/CrawlSwitchboard.java +++ b/source/net/yacy/crawler/CrawlSwitchboard.java @@ -36,8 +36,8 @@ import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.NaturalOrder; diff --git a/source/net/yacy/crawler/data/Cache.java b/source/net/yacy/crawler/data/Cache.java index ec08d3c68a..f1d72354f4 100644 --- a/source/net/yacy/crawler/data/Cache.java +++ b/source/net/yacy/crawler/data/Cache.java @@ -41,7 +41,8 @@ the class shall also be used to do a cache-cleaning and index creation import java.util.Map; import java.util.concurrent.BlockingQueue; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.ResponseHeader; import net.yacy.cora.storage.HandleSet; @@ -50,7 +51,6 @@ the class shall also be used to do a cache-cleaning and index creation import net.yacy.kelondro.blob.ArrayStack; import net.yacy.kelondro.blob.Compressor; import net.yacy.kelondro.blob.MapHeap; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.RowHandleSet; @@ -191,7 +191,7 @@ public static void close() { fileDB.close(true); } - public static void store(final DigestURI url, final ResponseHeader responseHeader, final byte[] file) throws IOException { + public static void store(final DigestURL url, final ResponseHeader responseHeader, final byte[] file) throws IOException { if (maxCacheSize == 0) return; if (responseHeader == null) throw new IOException("Cache.store of url " + url.toString() + " not possible: responseHeader == null"); if (file == null) throw new IOException("Cache.store of url " + url.toString() + " not possible: file == null"); diff --git a/source/net/yacy/crawler/data/CrawlProfile.java b/source/net/yacy/crawler/data/CrawlProfile.java index a07bdf1256..03efd21b9a 100644 --- a/source/net/yacy/crawler/data/CrawlProfile.java +++ b/source/net/yacy/crawler/data/CrawlProfile.java @@ -26,17 +26,17 @@ package net.yacy.crawler.data; import java.text.DateFormat; +import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.Digest; @@ -560,13 +560,13 @@ public static long getRecrawlDate(final long oldTimeMinutes) { return System.currentTimeMillis() - (60000L * oldTimeMinutes); } - public static String siteFilter(final Set uris) { + public static String siteFilter(final Collection uris) { final StringBuilder filter = new StringBuilder(); - for (final MultiProtocolURI uri: uris) filter.append('|').append(mustMatchFilterFullDomain(uri)); + for (final MultiProtocolURL uri: uris) filter.append('|').append(mustMatchFilterFullDomain(uri)); return filter.length() > 0 ? filter.substring(1) : CrawlProfile.MATCH_ALL_STRING; } - public static String mustMatchFilterFullDomain(final MultiProtocolURI uri) { + public static String mustMatchFilterFullDomain(final MultiProtocolURL uri) { String host = uri.getHost(); if (host == null) return uri.getProtocol() + ".*"; if (host.startsWith("www.")) host = host.substring(4); @@ -575,13 +575,13 @@ public static String mustMatchFilterFullDomain(final MultiProtocolURI uri) { return new StringBuilder(host.length() + 20).append(protocol).append("://(www.)?").append(Pattern.quote(host)).append(".*").toString(); } - public static String subpathFilter(final Set uris) { + public static String subpathFilter(final Collection uris) { final StringBuilder filter = new StringBuilder(); - for (final MultiProtocolURI uri: uris) filter.append('|').append(mustMatchSubpath(uri)); + for (final MultiProtocolURL uri: uris) filter.append('|').append(mustMatchSubpath(uri)); return filter.length() > 0 ? filter.substring(1) : CrawlProfile.MATCH_ALL_STRING; } - public static String mustMatchSubpath(final MultiProtocolURI uri) { + public static String mustMatchSubpath(final MultiProtocolURL uri) { String u = uri.toNormalform(true); if (!u.endsWith("/")) {int p = u.lastIndexOf("/"); if (p > 0) u = u.substring(0, p + 1);} return new StringBuilder(u.length() + 5).append(Pattern.quote(u)).append(".*").toString(); diff --git a/source/net/yacy/crawler/data/CrawlQueues.java b/source/net/yacy/crawler/data/CrawlQueues.java index bff9bce487..04ecfb924e 100644 --- a/source/net/yacy/crawler/data/CrawlQueues.java +++ b/source/net/yacy/crawler/data/CrawlQueues.java @@ -35,10 +35,11 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.Hit; -import net.yacy.cora.document.RSSFeed; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.feed.Hit; +import net.yacy.cora.document.feed.RSSFeed; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.ConnectionInfo; @@ -49,7 +50,6 @@ import net.yacy.crawler.retrieval.Request; import net.yacy.crawler.retrieval.Response; import net.yacy.crawler.robots.RobotsTxtEntry; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.workflow.WorkflowJob; import net.yacy.peers.DHTSelection; @@ -167,7 +167,7 @@ public void removeURL(final byte[] hash) { this.errorURL.remove(hash); } - public DigestURI getURL(final byte[] urlhash) { + public DigestURL getURL(final byte[] urlhash) { assert urlhash != null; if (urlhash == null || urlhash.length == 0) { return null; @@ -317,7 +317,7 @@ private void load(final Request urlEntry, final String stats, final String profi if (profile != null) { // check if the protocol is supported - final DigestURI url = urlEntry.url(); + final DigestURL url = urlEntry.url(); final String urlProtocol = url.getProtocol(); if (this.sb.loader.isSupportedProtocol(urlProtocol)) { if (this.log.isFine()) { @@ -502,19 +502,19 @@ public boolean remoteCrawlLoaderJob() { } // parse the rss - DigestURI url, referrer; + DigestURL url, referrer; Date loaddate; for (final Hit item: feed) { //System.out.println("URL=" + item.getLink() + ", desc=" + item.getDescription() + ", pubDate=" + item.getPubDate()); // put url on remote crawl stack try { - url = new DigestURI(item.getLink()); + url = new DigestURL(item.getLink()); } catch (final MalformedURLException e) { continue; } try { - referrer = new DigestURI(item.getReferrer()); + referrer = new DigestURL(item.getReferrer()); } catch (final MalformedURLException e) { referrer = null; } @@ -548,7 +548,7 @@ public boolean remoteCrawlLoaderJob() { * @param url * @return */ - private static String urlToString(final DigestURI url) { + private static String urlToString(final DigestURL url) { return (url == null ? "null" : url.toNormalform(true)); } diff --git a/source/net/yacy/crawler/data/Latency.java b/source/net/yacy/crawler/data/Latency.java index f6edbaca6f..34bd026ebb 100644 --- a/source/net/yacy/crawler/data/Latency.java +++ b/source/net/yacy/crawler/data/Latency.java @@ -29,11 +29,11 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.crawler.robots.RobotsTxt; import net.yacy.crawler.robots.RobotsTxtEntry; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.MemoryControl; @@ -50,7 +50,7 @@ public class Latency { * @param url * @param robotsCrawlDelay the crawl-delay given by the robots; 0 if not exist */ - public static void updateAfterSelection(final DigestURI url, final long robotsCrawlDelay) { + public static void updateAfterSelection(final DigestURL url, final long robotsCrawlDelay) { final String host = url.getHost(); if (host == null) return; String hosthash = url.hosthash(); @@ -67,7 +67,7 @@ public static void updateAfterSelection(final DigestURI url, final long robotsCr * @param url * @param time the time to load the file in milliseconds */ - public static void updateBeforeLoad(final DigestURI url) { + public static void updateBeforeLoad(final DigestURL url) { final String host = url.getHost(); if (host == null) return; String hosthash = url.hosthash(); @@ -86,7 +86,7 @@ public static void updateBeforeLoad(final DigestURI url) { * @param url * @param time the time to load the file in milliseconds */ - public static void updateAfterLoad(final DigestURI url, final long time) { + public static void updateAfterLoad(final DigestURL url, final long time) { final String host = url.getHost(); if (host == null) return; String hosthash = url.hosthash(); @@ -100,7 +100,7 @@ public static void updateAfterLoad(final DigestURI url, final long time) { } } - private static Host host(final DigestURI url) { + private static Host host(final DigestURL url) { final String host = url.getHost(); if (host == null) return null; return map.get(url.hosthash()); @@ -119,7 +119,7 @@ public static Iterator> iterator() { * @param thisAgents * @return the waiting time in milliseconds; 0 if not known; -1 if host gives us special rights */ - public static int waitingRobots(final MultiProtocolURI url, final RobotsTxt robots, final ClientIdentification.Agent agent) { + public static int waitingRobots(final MultiProtocolURL url, final RobotsTxt robots, final ClientIdentification.Agent agent) { int robotsDelay = 0; RobotsTxtEntry robotsEntry = robots.getEntry(url, agent); robotsDelay = (robotsEntry == null) ? 0 : robotsEntry.getCrawlDelayMillis(); @@ -187,7 +187,7 @@ public static int waitingRemainingGuessed(final String hostname, final String ho * @param agent * @return the remaining waiting time in milliseconds. can be negative to reflect the due-time after a possible nex loading time */ - public static int waitingRemaining(final DigestURI url, final RobotsTxt robots, final ClientIdentification.Agent agent) { + public static int waitingRemaining(final DigestURL url, final RobotsTxt robots, final ClientIdentification.Agent agent) { // first check if the domain was _ever_ accessed before final Host host = host(url); @@ -200,7 +200,7 @@ public static int waitingRemaining(final DigestURI url, final RobotsTxt robots, // for CGI accesses, we double the minimum time // mostly there is a database access in the background // which creates a lot of unwanted IO on target site - if (MultiProtocolURI.isCGI(url.getFileName())) waiting = waiting * 2; + if (MultiProtocolURL.isCGI(url.getFileName())) waiting = waiting * 2; // if we have accessed the domain many times, get slower (the flux factor) if (!local) waiting += host.flux(waiting); @@ -219,7 +219,7 @@ public static int waitingRemaining(final DigestURI url, final RobotsTxt robots, return Math.min(60000, waiting) - timeSinceLastAccess; } - public static String waitingRemainingExplain(final DigestURI url, final RobotsTxt robots, final ClientIdentification.Agent agent) { + public static String waitingRemainingExplain(final DigestURL url, final RobotsTxt robots, final ClientIdentification.Agent agent) { // first check if the domain was _ever_ accessed before final Host host = host(url); @@ -234,7 +234,7 @@ public static String waitingRemainingExplain(final DigestURI url, final RobotsTx // for CGI accesses, we double the minimum time // mostly there is a database access in the background // which creates a lot of unwanted IO on target site - if (MultiProtocolURI.isCGI(url.getFileName())) { waiting = waiting * 2; s.append(", isCGI = true -> double"); } + if (MultiProtocolURL.isCGI(url.getFileName())) { waiting = waiting * 2; s.append(", isCGI = true -> double"); } // if we have accessed the domain many times, get slower (the flux factor) int flux = host.flux(waiting); diff --git a/source/net/yacy/crawler/data/ResultImages.java b/source/net/yacy/crawler/data/ResultImages.java index 1d9fdab5a6..adeecd40c9 100644 --- a/source/net/yacy/crawler/data/ResultImages.java +++ b/source/net/yacy/crawler/data/ResultImages.java @@ -31,11 +31,11 @@ import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.storage.SizeLimitedSet; import net.yacy.document.Document; import net.yacy.document.parser.html.ImageEntry; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.MemoryControl; @@ -54,14 +54,14 @@ public class ResultImages { // the same images may be linked from different pages private static final Set doubleCheck = new SizeLimitedSet(10000); - public static void registerImages(final DigestURI source, final Document document, final boolean privateEntry) { + public static void registerImages(final DigestURL source, final Document document, final boolean privateEntry) { if (document == null) return; if (source == null) return; if (MemoryControl.shortStatus()) clearQueues(); limitQueues(1000); - final Map images = document.getImages(); + final Map images = document.getImages(); for (final ImageEntry image: images.values()) { // do a double-check; attention: this can be time-consuming since this possibly needs a DNS-lookup if (image == null || image.url() == null) continue; @@ -74,7 +74,7 @@ public static void registerImages(final DigestURI source, final Document documen image.height() > 100 && image.width() < 1200 && image.height() < 1000 && - !"gif".equals(MultiProtocolURI.getFileExtension(image.url().getFileName()))) { + !"gif".equals(MultiProtocolURL.getFileExtension(image.url().getFileName()))) { // && ((urlString.lastIndexOf(".jpg") != -1)) || // ((urlString.lastIndexOf(".png") != -1)){ @@ -158,8 +158,8 @@ public static void limitQueues(int limit) { public static class OriginEntry { public ImageEntry imageEntry; - public MultiProtocolURI baseURL; - public OriginEntry(final ImageEntry imageEntry, final MultiProtocolURI baseURL) { + public MultiProtocolURL baseURL; + public OriginEntry(final ImageEntry imageEntry, final MultiProtocolURL baseURL) { this.imageEntry = imageEntry; this.baseURL = baseURL; } diff --git a/source/net/yacy/crawler/data/ZURL.java b/source/net/yacy/crawler/data/ZURL.java index 2269073252..af0c47a4c0 100644 --- a/source/net/yacy/crawler/data/ZURL.java +++ b/source/net/yacy/crawler/data/ZURL.java @@ -37,14 +37,14 @@ import org.apache.solr.common.SolrInputDocument; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.solr.FailType; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.NaturalOrder; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; import net.yacy.crawler.retrieval.Request; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.Index; import net.yacy.kelondro.index.Row; @@ -211,7 +211,7 @@ public Iterator iterator() { public ArrayList list(int max) { final ArrayList l = new ArrayList(); - DigestURI url; + DigestURL url; for (final ZURL.Entry entry: this) { if (entry == null) continue; url = entry.url(); @@ -330,7 +330,7 @@ private Entry(final Row.Entry entry) throws IOException { return; } - public DigestURI url() { + public DigestURL url() { return this.bentry.url(); } diff --git a/source/net/yacy/crawler/retrieval/FTPLoader.java b/source/net/yacy/crawler/retrieval/FTPLoader.java index 080117d01e..148853636e 100644 --- a/source/net/yacy/crawler/retrieval/FTPLoader.java +++ b/source/net/yacy/crawler/retrieval/FTPLoader.java @@ -32,9 +32,10 @@ import java.io.PrintStream; import java.util.Date; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.ResponseHeader; @@ -44,7 +45,6 @@ import net.yacy.crawler.data.Latency; import net.yacy.crawler.data.ZURL.FailCategory; import net.yacy.document.TextParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; public class FTPLoader { @@ -72,7 +72,7 @@ public Response load(final Request request, final boolean acceptOnlyParseable) t Latency.updateBeforeLoad(request.url()); final long start = System.currentTimeMillis(); - final DigestURI entryUrl = request.url(); + final DigestURL entryUrl = request.url(); final String fullPath = getPath(entryUrl); // the return value @@ -119,7 +119,7 @@ public Response load(final Request request, final boolean acceptOnlyParseable) t // directory -> get list of files final RequestHeader requestHeader = new RequestHeader(); if (request.referrerhash() != null) { - final DigestURI u = this.sb.getURL(request.referrerhash()); + final DigestURL u = this.sb.getURL(request.referrerhash()); if (u != null) requestHeader.put(RequestHeader.REFERER, u.toNormalform(true)); } @@ -176,7 +176,7 @@ private void closeConnection(final FTPClient ftpClient) { /** * establish a connection to the ftp server (open, login, set transfer mode) */ - private boolean openConnection(final FTPClient ftpClient, final DigestURI entryUrl) { + private boolean openConnection(final FTPClient ftpClient, final DigestURL entryUrl) { // get username and password final String userInfo = entryUrl.getUserInfo(); String userName = "anonymous", userPwd = "anonymous"; @@ -215,7 +215,7 @@ private boolean openConnection(final FTPClient ftpClient, final DigestURI entryU private Response getFile(final FTPClient ftpClient, final Request request, final boolean acceptOnlyParseable) throws IOException { // determine the mimetype of the resource - final DigestURI url = request.url(); + final DigestURL url = request.url(); final String mime = TextParser.mimeOf(url); final String path = getPath(url); @@ -225,7 +225,7 @@ private Response getFile(final FTPClient ftpClient, final Request request, final // create response header final RequestHeader requestHeader = new RequestHeader(); if (request.referrerhash() != null) { - final DigestURI refurl = this.sb.getURL(request.referrerhash()); + final DigestURL refurl = this.sb.getURL(request.referrerhash()); if (refurl != null) requestHeader.put(RequestHeader.REFERER, refurl.toNormalform(true)); } final ResponseHeader responseHeader = new ResponseHeader(200); @@ -281,8 +281,8 @@ private Response getFile(final FTPClient ftpClient, final Request request, final * @param entryUrl * @return */ - private String getPath(final MultiProtocolURI entryUrl) { - return MultiProtocolURI.unescape(entryUrl.getPath()).replace("\"", "\"\""); + private String getPath(final MultiProtocolURL entryUrl) { + return MultiProtocolURL.unescape(entryUrl.getPath()).replace("\"", "\"\""); } } diff --git a/source/net/yacy/crawler/retrieval/FileLoader.java b/source/net/yacy/crawler/retrieval/FileLoader.java index f4aeb45769..06d8bde3c2 100644 --- a/source/net/yacy/crawler/retrieval/FileLoader.java +++ b/source/net/yacy/crawler/retrieval/FileLoader.java @@ -30,10 +30,11 @@ import java.util.Date; import java.util.List; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; import net.yacy.cora.document.analysis.Classification; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; @@ -42,7 +43,6 @@ import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.data.CrawlProfile; import net.yacy.document.TextParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.search.Switchboard; @@ -59,12 +59,12 @@ public FileLoader(final Switchboard sb, final ConcurrentLog log) { } public Response load(final Request request, boolean acceptOnlyParseable) throws IOException { - DigestURI url = request.url(); + DigestURL url = request.url(); if (!url.getProtocol().equals("file")) throw new IOException("wrong loader for FileLoader: " + url.getProtocol()); RequestHeader requestHeader = new RequestHeader(); if (request.referrerhash() != null) { - DigestURI ur = this.sb.getURL(request.referrerhash()); + DigestURL ur = this.sb.getURL(request.referrerhash()); if (ur != null) requestHeader.put(RequestHeader.REFERER, ur.toNormalform(true)); } @@ -96,7 +96,7 @@ public Response load(final Request request, boolean acceptOnlyParseable) throws } // create response header - String mime = Classification.ext2mime(MultiProtocolURI.getFileExtension(url.getFileName())); + String mime = Classification.ext2mime(MultiProtocolURL.getFileExtension(url.getFileName())); ResponseHeader responseHeader = new ResponseHeader(200); responseHeader.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(new Date(url.lastModified()))); responseHeader.put(HeaderFramework.CONTENT_TYPE, mime); diff --git a/source/net/yacy/crawler/retrieval/HTTPLoader.java b/source/net/yacy/crawler/retrieval/HTTPLoader.java index 248a9e8d58..2383cc1287 100644 --- a/source/net/yacy/crawler/retrieval/HTTPLoader.java +++ b/source/net/yacy/crawler/retrieval/HTTPLoader.java @@ -27,7 +27,8 @@ import java.io.IOException; import java.util.Date; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; @@ -37,7 +38,6 @@ import net.yacy.crawler.data.CrawlProfile; import net.yacy.crawler.data.Latency; import net.yacy.crawler.data.ZURL.FailCategory; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.io.ByteCount; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; @@ -86,7 +86,7 @@ private Response load(final Request request, CrawlProfile profile, final int ret throw new IOException("retry counter exceeded for URL " + request.url().toString() + ". Processing aborted."); } - DigestURI url = request.url(); + DigestURL url = request.url(); final String host = url.getHost(); if (host == null || host.length() < 2) throw new IOException("host is not well-formed: '" + host + "'"); @@ -107,7 +107,7 @@ private Response load(final Request request, CrawlProfile profile, final int ret if(yacyResolver != null) { final String yAddress = yacyResolver.resolve(host); if(yAddress != null) { - url = new DigestURI(url.getProtocol() + "://" + yAddress + path); + url = new DigestURL(url.getProtocol() + "://" + yAddress + path); } } @@ -117,7 +117,7 @@ private Response load(final Request request, CrawlProfile profile, final int ret // create a request header final RequestHeader requestHeader = new RequestHeader(); requestHeader.put(HeaderFramework.USER_AGENT, agent.userAgent); - DigestURI refererURL = null; + DigestURL refererURL = null; if (request.referrerhash() != null) refererURL = this.sb.getURL(request.referrerhash()); if (refererURL != null) requestHeader.put(RequestHeader.REFERER, refererURL.toNormalform(true)); requestHeader.put(HeaderFramework.ACCEPT, this.sb.getConfig("crawler.http.accept", DEFAULT_ACCEPT)); @@ -150,7 +150,7 @@ private Response load(final Request request, CrawlProfile profile, final int ret } // normalize URL - final DigestURI redirectionUrl = DigestURI.newURL(request.url(), redirectionUrlString); + final DigestURL redirectionUrl = DigestURL.newURL(request.url(), redirectionUrlString); // restart crawling with new url this.log.info("CRAWLER Redirection detected ('" + client.getHttpResponse().getStatusLine() + "') for URL " + requestURLString); @@ -283,7 +283,7 @@ private static Response load(final Request request, ClientIdentification.Agent a } // normalizing URL - final DigestURI redirectionUrl = DigestURI.newURL(request.url(), redirectionUrlString); + final DigestURL redirectionUrl = DigestURL.newURL(request.url(), redirectionUrlString); // if we are already doing a shutdown we don't need to retry crawling diff --git a/source/net/yacy/crawler/retrieval/RSSLoader.java b/source/net/yacy/crawler/retrieval/RSSLoader.java index 80dffa45a9..0394833865 100644 --- a/source/net/yacy/crawler/retrieval/RSSLoader.java +++ b/source/net/yacy/crawler/retrieval/RSSLoader.java @@ -33,11 +33,12 @@ import java.util.Map; import java.util.regex.Pattern; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.RSSFeed; -import net.yacy.cora.document.RSSMessage; -import net.yacy.cora.document.RSSReader; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.feed.RSSFeed; +import net.yacy.cora.document.feed.RSSMessage; +import net.yacy.cora.document.feed.RSSReader; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.ClientIdentification; @@ -48,7 +49,6 @@ import net.yacy.crawler.HarvestProcess; import net.yacy.data.WorkTables; import net.yacy.kelondro.blob.Tables; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; @@ -57,12 +57,12 @@ public class RSSLoader extends Thread { public static final ARC indexTriggered = new ComparableARC(1000, Base64Order.enhancedCoder); - private final DigestURI urlf; + private final DigestURL urlf; private final Switchboard sb; private final Map collections; private final ClientIdentification.Agent agent; - public RSSLoader(final Switchboard sb, final DigestURI urlf, final Map collections, final ClientIdentification.Agent agent) { + public RSSLoader(final Switchboard sb, final DigestURL urlf, final Map collections, final ClientIdentification.Agent agent) { this.sb = sb; this.urlf = urlf; this.collections = collections; @@ -94,13 +94,13 @@ public void run() { recordAPI(this.sb, null, this.urlf, feed, 7, "seldays"); } - public static void indexAllRssFeed(final Switchboard sb, final DigestURI url, final RSSFeed feed, Map collections) { + public static void indexAllRssFeed(final Switchboard sb, final DigestURL url, final RSSFeed feed, Map collections) { int loadCount = 0; - List list = new ArrayList(); - Map urlmap = new HashMap(); + List list = new ArrayList(); + Map urlmap = new HashMap(); for (final RSSMessage message: feed) { try { - final DigestURI messageurl = new DigestURI(message.getLink()); + final DigestURL messageurl = new DigestURL(message.getLink()); if (indexTriggered.containsKey(messageurl.hash())) continue; urlmap.put(ASCII.String(messageurl.hash()), messageurl); } catch (final IOException e) { @@ -108,7 +108,7 @@ public static void indexAllRssFeed(final Switchboard sb, final DigestURI url, fi } } Map existingids = sb.urlExists(urlmap.keySet()); - for (final Map.Entry e: urlmap.entrySet()) { + for (final Map.Entry e: urlmap.entrySet()) { if (existingids.get(e.getKey()) != null) continue; list.add(e.getValue()); indexTriggered.insertIfAbsent(ASCII.getBytes(e.getKey()), new Date()); @@ -141,7 +141,7 @@ public static void indexAllRssFeed(final Switchboard sb, final DigestURI url, fi } - public static void recordAPI(final Switchboard sb, final String apicall_pk, final DigestURI url, final RSSFeed feed, final int repeat_time, final String repeat_unit) { + public static void recordAPI(final Switchboard sb, final String apicall_pk, final DigestURL url, final RSSFeed feed, final int repeat_time, final String repeat_unit) { // record API action byte[] pk = null; final serverObjects post = new serverObjects(); diff --git a/source/net/yacy/crawler/retrieval/Request.java b/source/net/yacy/crawler/retrieval/Request.java index de8cfd5bb4..227a7f0d97 100644 --- a/source/net/yacy/crawler/retrieval/Request.java +++ b/source/net/yacy/crawler/retrieval/Request.java @@ -29,11 +29,11 @@ import java.io.IOException; import java.util.Date; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.NaturalOrder; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.util.Bitfield; @@ -84,7 +84,7 @@ public class Request extends WorkflowJob private byte[] initiator; // the initiator hash, is NULL or "" if it is the own proxy; // if this is generated by a crawl, the own peer hash in entered private byte[] refhash; // the url's referrer hash - private DigestURI url; // the url as string + private DigestURL url; // the url as string private String name; // the name of the url, from anchor tag name private long appdate; // the time when the url was first time appeared. private String profileHandle; // the name of the fetch profile @@ -102,7 +102,7 @@ public class Request extends WorkflowJob * @param url * @param referrerhash */ - public Request(final DigestURI url, final byte[] referrerhash) { + public Request(final DigestURL url, final byte[] referrerhash) { this(null, url, referrerhash, null, null, null, 0, 0, 0, 0); } @@ -121,7 +121,7 @@ public Request(final DigestURI url, final byte[] referrerhash) { */ public Request( final byte[] initiator, - final DigestURI url, + final DigestURL url, final byte[] referrerhash, final String name, final Date appdate, @@ -166,7 +166,7 @@ private void insertEntry(final Row.Entry entry) throws IOException { this.initiator = entry.getColBytes(1, true); this.initiator = (this.initiator == null) ? null : ((this.initiator.length == 0) ? null : this.initiator); - this.url = new DigestURI(urlstring, entry.getPrimaryKeyBytes()); + this.url = new DigestURL(urlstring, entry.getPrimaryKeyBytes()); this.refhash = (entry.empty(3)) ? null : entry.getColBytes(3, true); this.name = (entry.empty(4)) ? "" : entry.getColUTF8(4).trim(); this.appdate = entry.getColLong(5); @@ -230,12 +230,12 @@ public Row.Entry toRow() { return rowdef.newEntry(entry); } - public DigestURI url() { + public DigestURL url() { // the url return this.url; } - public void redirectURL(final DigestURI redirectedURL) { + public void redirectURL(final DigestURL redirectedURL) { // replace old URL by new one. This should only be used in case of url redirection this.url = redirectedURL; } diff --git a/source/net/yacy/crawler/retrieval/Response.java b/source/net/yacy/crawler/retrieval/Response.java index 0d19bb7631..717a653246 100644 --- a/source/net/yacy/crawler/retrieval/Response.java +++ b/source/net/yacy/crawler/retrieval/Response.java @@ -29,10 +29,11 @@ import java.util.Date; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; import net.yacy.cora.document.analysis.Classification; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.ResponseHeader; @@ -42,7 +43,6 @@ import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.TextParser; -import net.yacy.kelondro.data.meta.DigestURI; public class Response { @@ -69,8 +69,8 @@ public class Response { private final boolean fromCache; // doctype calculation - public static char docType(final MultiProtocolURI url) { - String ext = MultiProtocolURI.getFileExtension(url.getFileName()); + public static char docType(final MultiProtocolURL url) { + String ext = MultiProtocolURL.getFileExtension(url.getFileName()); if (ext == null) return DT_UNKNOWN; if (ext.equals(".gif")) return DT_IMAGE; if (ext.equals(".ico")) return DT_IMAGE; @@ -171,7 +171,7 @@ public Response(final Request request, final CrawlProfile profile) { // request and response headers may be zero in case that we process surrogates this.requestHeader = new RequestHeader(); this.responseHeader = new ResponseHeader(200); - this.responseHeader.put(HeaderFramework.CONTENT_TYPE, Classification.ext2mime(MultiProtocolURI.getFileExtension(request.url().getFileName()), "text/plain")); // tell parser how to handle the content + this.responseHeader.put(HeaderFramework.CONTENT_TYPE, Classification.ext2mime(MultiProtocolURL.getFileExtension(request.url().getFileName()), "text/plain")); // tell parser how to handle the content if (!request.isEmpty()) this.responseHeader.put(HeaderFramework.CONTENT_LENGTH, Long.toString(request.size())); this.profile = profile; this.status = QUEUE_STATE_FRESH; @@ -210,7 +210,7 @@ public String name() { return this.request.name(); } - public DigestURI url() { + public DigestURL url() { return this.request.url(); } @@ -293,7 +293,7 @@ public String shallStoreCacheForProxy() { return "dynamic_post"; } - if (MultiProtocolURI.isCGI(MultiProtocolURI.getFileExtension(url().getFileName()))) { + if (MultiProtocolURL.isCGI(MultiProtocolURL.getFileExtension(url().getFileName()))) { return "dynamic_cgi"; } @@ -392,7 +392,7 @@ public boolean isFreshForProxy() { if (url().isPOST()) { return false; } - if (MultiProtocolURI.isCGI(MultiProtocolURI.getFileExtension(url().getFileName()))) { + if (MultiProtocolURL.isCGI(MultiProtocolURL.getFileExtension(url().getFileName()))) { return false; } @@ -543,7 +543,7 @@ public final String shallIndexCacheForProxy() { if (url().isPOST()) { return "Dynamic_(POST)"; } - if (MultiProtocolURI.isCGI(MultiProtocolURI.getFileExtension(url().getFileName()))) { + if (MultiProtocolURL.isCGI(MultiProtocolURL.getFileExtension(url().getFileName()))) { return "Dynamic_(CGI)"; } } @@ -686,7 +686,7 @@ public final String shallIndexCacheForCrawler() { // CGI access makes the page very individual, and therefore not usable in caches if (!profile().crawlingQ()) { if (url().isPOST()) { return "Dynamic_(POST)"; } - if (MultiProtocolURI.isCGI(MultiProtocolURI.getFileExtension(url().getFileName()))) { return "Dynamic_(CGI)"; } + if (MultiProtocolURL.isCGI(MultiProtocolURL.getFileExtension(url().getFileName()))) { return "Dynamic_(CGI)"; } } // -authorization cases in request @@ -759,12 +759,12 @@ public String getCharacterEncoding() { return this.responseHeader.getCharacterEncoding(); } - public DigestURI referrerURL() { + public DigestURL referrerURL() { if (this.requestHeader == null) return null; try { final String r = this.requestHeader.get(RequestHeader.REFERER, null); if (r == null) return null; - return new DigestURI(r); + return new DigestURL(r); } catch (final Exception e) { return null; } @@ -775,7 +775,7 @@ public byte[] referrerHash() { final String u = this.requestHeader.get(RequestHeader.REFERER, ""); if (u == null || u.isEmpty()) return null; try { - return new DigestURI(u).hash(); + return new DigestURL(u).hash(); } catch (final Exception e) { return null; } diff --git a/source/net/yacy/crawler/retrieval/SMBLoader.java b/source/net/yacy/crawler/retrieval/SMBLoader.java index f8b87d4f11..56c2adca13 100644 --- a/source/net/yacy/crawler/retrieval/SMBLoader.java +++ b/source/net/yacy/crawler/retrieval/SMBLoader.java @@ -38,10 +38,11 @@ import jcifs.smb.SmbException; import jcifs.smb.SmbFile; import jcifs.smb.SmbFileInputStream; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; import net.yacy.cora.document.analysis.Classification; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; @@ -50,7 +51,6 @@ import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.data.CrawlProfile; import net.yacy.document.TextParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.search.Switchboard; @@ -70,12 +70,12 @@ public SMBLoader(final Switchboard sb, final ConcurrentLog log) { public Response load(final Request request, boolean acceptOnlyParseable) throws IOException { - DigestURI url = request.url(); + DigestURL url = request.url(); if (!url.getProtocol().equals("smb")) throw new IOException("wrong loader for SMBLoader: " + url.getProtocol()); RequestHeader requestHeader = new RequestHeader(); if (request.referrerhash() != null) { - DigestURI ur = this.sb.getURL(request.referrerhash()); + DigestURL ur = this.sb.getURL(request.referrerhash()); if (ur != null) requestHeader.put(RequestHeader.REFERER, ur.toNormalform(true)); } @@ -87,7 +87,7 @@ public Response load(final Request request, boolean acceptOnlyParseable) throws List list = new ArrayList(); for (String s: l) { if (s.startsWith(".")) continue; - s = MultiProtocolURI.escape(s).toString(); + s = MultiProtocolURL.escape(s).toString(); if (!s.endsWith("/") && !s.endsWith("\\")) { // check if this is a directory SmbFile sf = new SmbFile(u + s); @@ -114,7 +114,7 @@ public Response load(final Request request, boolean acceptOnlyParseable) throws } // create response header - String mime = Classification.ext2mime(MultiProtocolURI.getFileExtension(url.getFileName())); + String mime = Classification.ext2mime(MultiProtocolURL.getFileExtension(url.getFileName())); ResponseHeader responseHeader = new ResponseHeader(200); responseHeader.put(HeaderFramework.LAST_MODIFIED, HeaderFramework.formatRFC1123(new Date(url.lastModified()))); responseHeader.put(HeaderFramework.CONTENT_TYPE, mime); diff --git a/source/net/yacy/crawler/retrieval/SitemapImporter.java b/source/net/yacy/crawler/retrieval/SitemapImporter.java index e6a7b88a1c..55ea340d33 100644 --- a/source/net/yacy/crawler/retrieval/SitemapImporter.java +++ b/source/net/yacy/crawler/retrieval/SitemapImporter.java @@ -28,13 +28,13 @@ import java.net.MalformedURLException; import java.util.Date; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.HarvestProcess; import net.yacy.crawler.data.CrawlProfile; import net.yacy.document.parser.sitemapParser; import net.yacy.document.parser.sitemapParser.URLEntry; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.search.Switchboard; @@ -42,10 +42,10 @@ public class SitemapImporter extends Thread { private CrawlProfile crawlingProfile = null; private static final ConcurrentLog logger = new ConcurrentLog("SITEMAP"); - private DigestURI siteMapURL = null; + private DigestURL siteMapURL = null; private final Switchboard sb; - public SitemapImporter(final Switchboard sb, final DigestURI sitemapURL, final CrawlProfile profileEntry) { + public SitemapImporter(final Switchboard sb, final DigestURL sitemapURL, final CrawlProfile profileEntry) { assert sitemapURL != null; this.sb = sb; this.siteMapURL = sitemapURL; @@ -72,9 +72,9 @@ public void process(sitemapParser.URLEntry entry) { // get the url hash byte[] nexturlhash = null; - DigestURI url = null; + DigestURL url = null; try { - url = new DigestURI(entry.url()); + url = new DigestURL(entry.url()); nexturlhash = url.hash(); } catch (final MalformedURLException e1) { } diff --git a/source/net/yacy/crawler/robots/RobotsTxt.java b/source/net/yacy/crawler/robots/RobotsTxt.java index 61074d89aa..a109bf5052 100644 --- a/source/net/yacy/crawler/robots/RobotsTxt.java +++ b/source/net/yacy/crawler/robots/RobotsTxt.java @@ -35,7 +35,8 @@ import java.util.concurrent.ConcurrentMap; import java.util.regex.Pattern; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.HeaderFramework; @@ -45,7 +46,6 @@ import net.yacy.crawler.retrieval.Response; import net.yacy.data.WorkTables; import net.yacy.kelondro.blob.BEncodedHeap; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.repository.LoaderDispatcher; public class RobotsTxt { @@ -89,7 +89,7 @@ public int size() throws IOException { return this.tables.getHeap(WorkTables.TABLE_ROBOTS_NAME).size(); } - public RobotsTxtEntry getEntry(final MultiProtocolURI theURL, final ClientIdentification.Agent agent) { + public RobotsTxtEntry getEntry(final MultiProtocolURL theURL, final ClientIdentification.Agent agent) { if (theURL == null) throw new IllegalArgumentException(); if (!theURL.getProtocol().startsWith("http")) return null; return getEntry(getHostPort(theURL), agent, true); @@ -151,9 +151,9 @@ record = null; } // generating the proper url to download the robots txt - DigestURI robotsURL = null; + DigestURL robotsURL = null; try { - robotsURL = new DigestURI((urlHostPort.endsWith(":443") ? "https://" : "http://") + urlHostPort + "/robots.txt"); + robotsURL = new DigestURL((urlHostPort.endsWith(":443") ? "https://" : "http://") + urlHostPort + "/robots.txt"); } catch (final MalformedURLException e) { log.severe("Unable to generate robots.txt URL for host:port '" + urlHostPort + "'.", e); robotsURL = null; @@ -182,7 +182,7 @@ record = null; return robotsTxt4Host; } - public void ensureExist(final MultiProtocolURI theURL, final ClientIdentification.Agent agent, boolean concurrent) { + public void ensureExist(final MultiProtocolURL theURL, final ClientIdentification.Agent agent, boolean concurrent) { if (theURL.isLocal()) return; final String urlHostPort = getHostPort(theURL); if (urlHostPort == null) return; @@ -207,9 +207,9 @@ public void run(){ if (robotsTable.containsKey(robotsTable.encodedKey(urlHostPort))) return; // generating the proper url to download the robots txt - DigestURI robotsURL = null; + DigestURL robotsURL = null; try { - robotsURL = new DigestURI((urlHostPort.endsWith(":443") ? "https://" : "http://") + urlHostPort + "/robots.txt"); + robotsURL = new DigestURL((urlHostPort.endsWith(":443") ? "https://" : "http://") + urlHostPort + "/robots.txt"); } catch (final MalformedURLException e) { log.severe("Unable to generate robots.txt URL for host:port '" + urlHostPort + "'.", e); robotsURL = null; @@ -237,7 +237,7 @@ public void run(){ if (concurrent) t.start(); else t.run(); } - private void processOldEntry(RobotsTxtEntry robotsTxt4Host, DigestURI robotsURL, BEncodedHeap robotsTable) { + private void processOldEntry(RobotsTxtEntry robotsTxt4Host, DigestURL robotsURL, BEncodedHeap robotsTable) { // no robots.txt available, make an entry to prevent that the robots loading is done twice if (robotsTxt4Host == null) { // generate artificial entry @@ -265,7 +265,7 @@ private void processOldEntry(RobotsTxtEntry robotsTxt4Host, DigestURI robotsURL, } } - private void processNewEntry(DigestURI robotsURL, Response response, final String[] thisAgents) { + private void processNewEntry(DigestURL robotsURL, Response response, final String[] thisAgents) { final byte[] robotsTxt = response.getContent(); //Log.logInfo("RobotsTxt", "robots of " + robotsURL.toNormalform(true, true) + ":\n" + ((robotsTxt == null) ? "null" : UTF8.String(robotsTxt))); // debug TODO remove RobotsTxtParser parserResult; @@ -309,7 +309,7 @@ private String addEntry(final RobotsTxtEntry entry) { } } - static final String getHostPort(final MultiProtocolURI theURL) { + static final String getHostPort(final MultiProtocolURL theURL) { int port = theURL.getPort(); if (port == -1) { if (theURL.getProtocol().equalsIgnoreCase("http")) { diff --git a/source/net/yacy/crawler/robots/RobotsTxtEntry.java b/source/net/yacy/crawler/robots/RobotsTxtEntry.java index 8bc3fc4046..293f9b1277 100644 --- a/source/net/yacy/crawler/robots/RobotsTxtEntry.java +++ b/source/net/yacy/crawler/robots/RobotsTxtEntry.java @@ -36,9 +36,9 @@ import java.util.List; import java.util.Map; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.util.ByteArray; @@ -94,7 +94,7 @@ protected RobotsTxtEntry(final String hostName, final Map mem) { } protected RobotsTxtEntry( - final MultiProtocolURI theURL, + final MultiProtocolURL theURL, final List allowPathList, final List disallowPathList, final Date loadedDate, @@ -177,11 +177,11 @@ public String toString() { * get the sitemap url * @return the sitemap url or null if no sitemap url is given */ - public MultiProtocolURI getSitemap() { + public MultiProtocolURL getSitemap() { final String url = this.mem.containsKey(SITEMAP)? UTF8.String(this.mem.get(SITEMAP)): null; if (url == null) return null; try { - return new MultiProtocolURI(url); + return new MultiProtocolURL(url); } catch (final MalformedURLException e) { return null; } @@ -228,7 +228,7 @@ public int getCrawlDelayMillis() { return 0; } - public boolean isDisallowed(final MultiProtocolURI subpathURL) { + public boolean isDisallowed(final MultiProtocolURL subpathURL) { String path = subpathURL.getFile(); if (this.mem == null) { this.info = "no robots file available"; diff --git a/source/net/yacy/crawler/robots/RobotsTxtParser.java b/source/net/yacy/crawler/robots/RobotsTxtParser.java index dc589ea2e6..a5e54939e4 100644 --- a/source/net/yacy/crawler/robots/RobotsTxtParser.java +++ b/source/net/yacy/crawler/robots/RobotsTxtParser.java @@ -38,7 +38,7 @@ import java.util.ArrayList; import java.util.regex.Pattern; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; /* * A class for Parsing robots.txt files. diff --git a/source/net/yacy/data/BlogBoard.java b/source/net/yacy/data/BlogBoard.java index 303081776a..c1ec79f154 100644 --- a/source/net/yacy/data/BlogBoard.java +++ b/source/net/yacy/data/BlogBoard.java @@ -48,7 +48,7 @@ import javax.xml.parsers.ParserConfigurationException; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.NaturalOrder; import net.yacy.cora.protocol.Domains; diff --git a/source/net/yacy/data/BlogBoardComments.java b/source/net/yacy/data/BlogBoardComments.java index 719e0cf386..9c1e03ba9f 100644 --- a/source/net/yacy/data/BlogBoardComments.java +++ b/source/net/yacy/data/BlogBoardComments.java @@ -45,7 +45,7 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.NaturalOrder; import net.yacy.cora.protocol.Domains; diff --git a/source/net/yacy/data/BookmarkDate.java b/source/net/yacy/data/BookmarkDate.java index 15d9ea6cdd..4c6df65d49 100644 --- a/source/net/yacy/data/BookmarkDate.java +++ b/source/net/yacy/data/BookmarkDate.java @@ -37,8 +37,7 @@ import java.util.List; import java.util.Map; - -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.NaturalOrder; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; diff --git a/source/net/yacy/data/BookmarkHelper.java b/source/net/yacy/data/BookmarkHelper.java index ecdc33cdb7..2d9dfd91fc 100644 --- a/source/net/yacy/data/BookmarkHelper.java +++ b/source/net/yacy/data/BookmarkHelper.java @@ -32,13 +32,11 @@ import java.io.Writer; import java.net.MalformedURLException; import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collection; import java.util.Date; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Properties; import java.util.Set; import java.util.TreeSet; @@ -47,14 +45,15 @@ import javax.xml.parsers.ParserConfigurationException; import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.AnchorURL; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.BookmarksDB.Bookmark; import net.yacy.data.BookmarksDB.Tag; import net.yacy.document.parser.html.ContentScraper; import net.yacy.document.parser.html.TransformerWriter; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.util.FileUtils; @@ -116,7 +115,7 @@ private static String tagHash(final String tagName, final String user){ // bookmarksDB's Import/Export functions // -------------------------------------- - public static int importFromBookmarks(final BookmarksDB db, final DigestURI baseURL, final String input, final String tag, final boolean importPublic){ + public static int importFromBookmarks(final BookmarksDB db, final DigestURL baseURL, final String input, final String tag, final boolean importPublic){ try { // convert string to input stream final ByteArrayInputStream byteIn = new ByteArrayInputStream(UTF8.getBytes(input)); @@ -129,13 +128,12 @@ public static int importFromBookmarks(final BookmarksDB db, final DigestURI base } } - private static int importFromBookmarks(final BookmarksDB db, final DigestURI baseURL, final InputStreamReader input, final String tag, final boolean importPublic){ + private static int importFromBookmarks(final BookmarksDB db, final DigestURL baseURL, final InputStreamReader input, final String tag, final boolean importPublic){ int importCount = 0; - Map links = new HashMap(); + Collection links = new ArrayList(); String title; - DigestURI url; Bookmark bm; final Set tags=ListManager.string2set(tag); //this allow multiple default tags try { @@ -147,9 +145,8 @@ private static int importFromBookmarks(final BookmarksDB db, final DigestURI bas writer.close(); links = scraper.getAnchors(); } catch (final IOException e) { ConcurrentLog.warn("BOOKMARKS", "error during load of links: "+ e.getClass() +" "+ e.getMessage());} - for (final Entry link: links.entrySet()) { - url = link.getKey(); - title = link.getValue().getProperty("name", ""); + for (final DigestURL url: links) { + title = url.getProperties().getProperty("name", ""); ConcurrentLog.info("BOOKMARKS", "links.get(url)"); if ("".equals(title)) {//cannot be displayed title = url.toString(); diff --git a/source/net/yacy/data/BookmarksDB.java b/source/net/yacy/data/BookmarksDB.java index c078d57aaf..1c11b4b15a 100644 --- a/source/net/yacy/data/BookmarksDB.java +++ b/source/net/yacy/data/BookmarksDB.java @@ -36,12 +36,12 @@ import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.order.NaturalOrder; import net.yacy.cora.util.ConcurrentLog; import net.yacy.kelondro.blob.MapHeap; -import net.yacy.kelondro.data.meta.DigestURI; public class BookmarksDB { @@ -502,7 +502,7 @@ public Bookmark(final String urlHash, final Map map) { loadTimestamp(); } - public Bookmark(final DigestURI url) { + public Bookmark(final DigestURL url) { this.entry = new HashMap(); this.urlHash = ASCII.String(url.hash()); this.entry.put(BOOKMARK_URL, url.toNormalform(false)); @@ -526,11 +526,11 @@ public Bookmark(final DigestURI url) { } public Bookmark(final String url) throws MalformedURLException { - this(new DigestURI((url.indexOf("://") < 0) ? "http://" + url : url)); + this(new DigestURL((url.indexOf("://") < 0) ? "http://" + url : url)); } public Bookmark(final Map map) throws MalformedURLException { - this(ASCII.String((new DigestURI(map.get(BOOKMARK_URL))).hash()), map); + this(ASCII.String((new DigestURL(map.get(BOOKMARK_URL))).hash()), map); } Map toMap() { diff --git a/source/net/yacy/data/MessageBoard.java b/source/net/yacy/data/MessageBoard.java index 614e0ca064..b7eab1cff8 100644 --- a/source/net/yacy/data/MessageBoard.java +++ b/source/net/yacy/data/MessageBoard.java @@ -32,7 +32,7 @@ import java.util.Map; import java.util.TimeZone; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.NaturalOrder; import net.yacy.cora.util.ConcurrentLog; diff --git a/source/net/yacy/data/URLLicense.java b/source/net/yacy/data/URLLicense.java index 63153fabeb..2f0f2cded9 100644 --- a/source/net/yacy/data/URLLicense.java +++ b/source/net/yacy/data/URLLicense.java @@ -29,9 +29,9 @@ import java.util.Collections; import java.util.Map; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.storage.SizeLimitedMap; -import net.yacy.kelondro.data.meta.DigestURI; public class URLLicense { @@ -41,7 +41,7 @@ public class URLLicense { private static final int maxQueue = 10000; private static final Map permissions = Collections.synchronizedMap(new SizeLimitedMap(maxQueue)); - public static String aquireLicense(final DigestURI url) { + public static String aquireLicense(final DigestURL url) { if (url == null) return ""; // generate license key String license = ASCII.String(url.hash()); diff --git a/source/net/yacy/data/UserDB.java b/source/net/yacy/data/UserDB.java index 584acddf57..8659e08058 100644 --- a/source/net/yacy/data/UserDB.java +++ b/source/net/yacy/data/UserDB.java @@ -36,7 +36,7 @@ import java.util.Map; import java.util.Random; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.order.Digest; diff --git a/source/net/yacy/data/WorkTables.java b/source/net/yacy/data/WorkTables.java index d237ac3fdb..0c3692393a 100644 --- a/source/net/yacy/data/WorkTables.java +++ b/source/net/yacy/data/WorkTables.java @@ -39,8 +39,9 @@ import java.util.TreeMap; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.http.HTTPClient; @@ -49,7 +50,6 @@ import net.yacy.cora.util.SpaceExceededException; import net.yacy.data.ymark.YMarkTables; import net.yacy.kelondro.blob.Tables; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.rwi.IndexCell; import net.yacy.search.Switchboard; @@ -323,7 +323,7 @@ public static void calculateAPIScheduler(Tables.Data row, boolean update) { row.put(WorkTables.TABLE_API_COL_DATE_NEXT_EXEC, new Date(d)); } - public void failURLsRegisterMissingWord(IndexCell indexCell, final DigestURI url, HandleSet queryHashes, final String reason) { + public void failURLsRegisterMissingWord(IndexCell indexCell, final DigestURL url, HandleSet queryHashes, final String reason) { // remove words from index if (indexCell != null) { diff --git a/source/net/yacy/data/wiki/WikiBoard.java b/source/net/yacy/data/wiki/WikiBoard.java index 4d2f16e711..37998529a0 100644 --- a/source/net/yacy/data/wiki/WikiBoard.java +++ b/source/net/yacy/data/wiki/WikiBoard.java @@ -35,8 +35,8 @@ import java.util.Map; import java.util.TimeZone; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.NaturalOrder; import net.yacy.cora.protocol.Domains; diff --git a/source/net/yacy/data/ymark/TablesRowComparator.java b/source/net/yacy/data/ymark/TablesRowComparator.java index 0d51e07ea5..5bae5a61d6 100644 --- a/source/net/yacy/data/ymark/TablesRowComparator.java +++ b/source/net/yacy/data/ymark/TablesRowComparator.java @@ -2,7 +2,7 @@ import java.util.Comparator; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.kelondro.blob.Tables; public class TablesRowComparator implements Comparator { diff --git a/source/net/yacy/data/ymark/YMarkAutoTagger.java b/source/net/yacy/data/ymark/YMarkAutoTagger.java index 2d6bd1bed1..2b35a08fac 100644 --- a/source/net/yacy/data/ymark/YMarkAutoTagger.java +++ b/source/net/yacy/data/ymark/YMarkAutoTagger.java @@ -10,7 +10,8 @@ import java.util.TreeSet; import java.util.concurrent.ArrayBlockingQueue; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.util.ConcurrentLog; @@ -21,7 +22,6 @@ import net.yacy.document.Parser.Failure; import net.yacy.document.SentenceReader; import net.yacy.document.WordTokenizer; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; import net.yacy.repository.LoaderDispatcher; @@ -60,10 +60,10 @@ public YMarkAutoTagger(final LoaderDispatcher loader, final YMarkTables ymarks, } private static Document loadDocument(final String url, final LoaderDispatcher loader, ClientIdentification.Agent agent) throws IOException { - DigestURI uri; + DigestURL uri; Response response; try { - uri = new DigestURI(url); + uri = new DigestURL(url); } catch (final MalformedURLException e) { ConcurrentLog.warn(YMarkTables.BOOKMARKS_LOG, "loadDocument failed due to malformed url: "+url); return null; @@ -161,7 +161,7 @@ else if (token.length()>3) { } final String clean = YMarkUtil.cleanTagsString(buffer.toString()); if(clean.equals(YMarkEntry.BOOKMARK.TAGS.deflt())) { - return MultiProtocolURI.getFileExtension(document.dc_source().getFileName()); + return MultiProtocolURL.getFileExtension(document.dc_source().getFileName()); } return clean; } finally { diff --git a/source/net/yacy/data/ymark/YMarkCrawlStart.java b/source/net/yacy/data/ymark/YMarkCrawlStart.java index d545e6f6c2..1eea74ea96 100644 --- a/source/net/yacy/data/ymark/YMarkCrawlStart.java +++ b/source/net/yacy/data/ymark/YMarkCrawlStart.java @@ -32,7 +32,8 @@ import java.util.Iterator; import java.util.regex.Pattern; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.crawler.CrawlSwitchboard; @@ -40,7 +41,6 @@ import net.yacy.crawler.retrieval.Request; import net.yacy.data.WorkTables; import net.yacy.kelondro.blob.Tables; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; public class YMarkCrawlStart extends HashMap{ @@ -162,7 +162,7 @@ private void load() { protected static String crawlStart( final Switchboard sb, - final DigestURI startURL, + final DigestURL startURL, final String urlMustMatch, final String urlMustNotMatch, final int depth, diff --git a/source/net/yacy/data/ymark/YMarkDate.java b/source/net/yacy/data/ymark/YMarkDate.java index 34d8f876a8..a1ba233884 100644 --- a/source/net/yacy/data/ymark/YMarkDate.java +++ b/source/net/yacy/data/ymark/YMarkDate.java @@ -31,7 +31,7 @@ import java.util.Date; import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; public class YMarkDate { diff --git a/source/net/yacy/data/ymark/YMarkEntry.java b/source/net/yacy/data/ymark/YMarkEntry.java index 0ed54adc86..3b6162cb21 100644 --- a/source/net/yacy/data/ymark/YMarkEntry.java +++ b/source/net/yacy/data/ymark/YMarkEntry.java @@ -33,11 +33,11 @@ import java.util.Map; import java.util.TreeMap; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.data.CrawlProfile; import net.yacy.document.content.DCEntry; import net.yacy.kelondro.blob.Tables; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; public class YMarkEntry extends TreeMap { @@ -237,7 +237,7 @@ public Tables.Data getData() { } public void crawl(final YMarkCrawlStart.CRAWLSTART type, final boolean medialink, final Switchboard sb) throws MalformedURLException { - final DigestURI url = new DigestURI(this.get(BOOKMARK.URL.key())); + final DigestURL url = new DigestURL(this.get(BOOKMARK.URL.key())); switch(type) { case SINGLE: YMarkCrawlStart.crawlStart(sb, url, CrawlProfile.MATCH_ALL_STRING, CrawlProfile.MATCH_NEVER_STRING, 0, true, medialink); diff --git a/source/net/yacy/data/ymark/YMarkMetadata.java b/source/net/yacy/data/ymark/YMarkMetadata.java index 5814f5c6e3..3bc88dd7b0 100644 --- a/source/net/yacy/data/ymark/YMarkMetadata.java +++ b/source/net/yacy/data/ymark/YMarkMetadata.java @@ -31,19 +31,19 @@ import java.util.EnumMap; import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.crawler.retrieval.Response; import net.yacy.document.Document; import net.yacy.document.Parser.Failure; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.repository.LoaderDispatcher; import net.yacy.search.index.Segment; public class YMarkMetadata { - private DigestURI uri; + private DigestURL uri; Document document; Segment indexSegment; @@ -67,13 +67,13 @@ public enum METADATA { AUTOTAG } - public YMarkMetadata(final DigestURI uri) { + public YMarkMetadata(final DigestURL uri) { this.uri = uri; this.document = null; this.indexSegment = null; } - public YMarkMetadata(final DigestURI uri, final Segment indexSegment) { + public YMarkMetadata(final DigestURL uri, final Segment indexSegment) { this.uri = uri; this.document = null; this.indexSegment = indexSegment; @@ -88,7 +88,7 @@ public YMarkMetadata(final byte[] urlHash, final Segment indexSegment) { public YMarkMetadata(final Document document) { this.document = document; try { - this.uri = new DigestURI(this.document.dc_identifier()); + this.uri = new DigestURL(this.document.dc_identifier()); } catch (final MalformedURLException e) { this.uri = null; } diff --git a/source/net/yacy/data/ymark/YMarkRDF.java b/source/net/yacy/data/ymark/YMarkRDF.java index 5e07961413..95b4371e4a 100644 --- a/source/net/yacy/data/ymark/YMarkRDF.java +++ b/source/net/yacy/data/ymark/YMarkRDF.java @@ -6,7 +6,7 @@ import java.util.Iterator; import java.util.Map; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.lod.vocabulary.AnnoteaA; import net.yacy.cora.lod.vocabulary.AnnoteaB; import net.yacy.cora.lod.vocabulary.DCElements; diff --git a/source/net/yacy/data/ymark/YMarkTables.java b/source/net/yacy/data/ymark/YMarkTables.java index 3811191c5d..673c6ed7c6 100644 --- a/source/net/yacy/data/ymark/YMarkTables.java +++ b/source/net/yacy/data/ymark/YMarkTables.java @@ -42,7 +42,8 @@ import javax.swing.event.ChangeEvent; import javax.swing.event.ChangeListener; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; @@ -53,7 +54,6 @@ import net.yacy.kelondro.blob.Tables; import net.yacy.kelondro.blob.Tables.Row; import net.yacy.kelondro.blob.TablesColumnIndex; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.repository.LoaderDispatcher; public class YMarkTables { @@ -364,10 +364,10 @@ public void visited(final String bmk_user, final String url) throws IOException } public void createBookmark(final LoaderDispatcher loader, final String url, final ClientIdentification.Agent agent, final String bmk_user, final boolean autotag, final String tagsString, final String foldersString) throws IOException, Failure { - createBookmark(loader, new DigestURI(url), agent, bmk_user, autotag, tagsString, foldersString); + createBookmark(loader, new DigestURL(url), agent, bmk_user, autotag, tagsString, foldersString); } - public void createBookmark(final LoaderDispatcher loader, final DigestURI url, final ClientIdentification.Agent agent, final String bmk_user, final boolean autotag, final String tagsString, final String foldersString) throws IOException, Failure { + public void createBookmark(final LoaderDispatcher loader, final DigestURL url, final ClientIdentification.Agent agent, final String bmk_user, final boolean autotag, final String tagsString, final String foldersString) throws IOException, Failure { final YMarkEntry bmk_entry = new YMarkEntry(false); final YMarkMetadata meta = new YMarkMetadata(url); diff --git a/source/net/yacy/data/ymark/YMarkUtil.java b/source/net/yacy/data/ymark/YMarkUtil.java index 4ce2775063..e9a19f7eb5 100644 --- a/source/net/yacy/data/ymark/YMarkUtil.java +++ b/source/net/yacy/data/ymark/YMarkUtil.java @@ -30,8 +30,8 @@ import java.util.HashSet; import java.util.Iterator; -import net.yacy.cora.document.UTF8; -import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.kelondro.data.word.Word; public class YMarkUtil { @@ -48,7 +48,7 @@ public class YMarkUtil { * @see net.yacy.kelondro.data.meta.DigestURI.DigestURI(String url, byte[] hash).hash() */ public final static byte[] getBookmarkId(String url) throws MalformedURLException { - return (new DigestURI(url)).hash(); + return (new DigestURL(url)).hash(); } /** diff --git a/source/net/yacy/dbtest.java b/source/net/yacy/dbtest.java index f6fb0ce728..de1f68c8bb 100644 --- a/source/net/yacy/dbtest.java +++ b/source/net/yacy/dbtest.java @@ -11,7 +11,7 @@ import javax.imageio.ImageIO; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.order.NaturalOrder; diff --git a/source/net/yacy/document/Condenser.java b/source/net/yacy/document/Condenser.java index 86dde35610..bceda9b8c5 100644 --- a/source/net/yacy/document/Condenser.java +++ b/source/net/yacy/document/Condenser.java @@ -39,18 +39,18 @@ import org.apache.solr.common.params.MapSolrParams; -import net.yacy.cora.document.ASCII; import net.yacy.cora.document.WordCache; import net.yacy.cora.document.analysis.Classification.ContentDomain; import net.yacy.cora.document.analysis.EnhancedTextProfileSignature; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.solr.Ranking; import net.yacy.cora.language.synonyms.SynonymLibrary; import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.language.Identificator; import net.yacy.document.parser.html.ImageEntry; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.WordReferenceRow; import net.yacy.kelondro.util.Bitfield; @@ -113,7 +113,7 @@ public Condenser( // add the URL components to the word list insertTextToWords(new SentenceReader(document.dc_source().toTokens()), 0, WordReferenceRow.flag_app_dc_identifier, this.RESULT_FLAGS, false, meaningLib); - Map.Entry entry; + Map.Entry entry; if (indexText) { createCondensement(document.getTextString(), meaningLib, doAutotagging); // the phrase counter: @@ -165,7 +165,7 @@ public Condenser( if (indexMedia) { // add anchor descriptions: here, we also add the url components // audio - Iterator> i = document.getAudiolinks().entrySet().iterator(); + Iterator> i = document.getAudiolinks().entrySet().iterator(); while (i.hasNext()) { entry = i.next(); insertTextToWords(new SentenceReader(entry.getKey().toNormalform(true)), 99, flag_cat_hasaudio, this.RESULT_FLAGS, false, meaningLib); @@ -191,7 +191,7 @@ public Condenser( // images final Iterator j = document.getImages().values().iterator(); ImageEntry ientry; - MultiProtocolURI url; + MultiProtocolURL url; while (j.hasNext()) { ientry = j.next(); url = ientry.url(); diff --git a/source/net/yacy/document/Document.java b/source/net/yacy/document/Document.java index 6fada35c9e..e8e06e51bc 100644 --- a/source/net/yacy/document/Document.java +++ b/source/net/yacy/document/Document.java @@ -5,10 +5,6 @@ //first published on http://www.anomic.de //Frankfurt, Germany, 2005 // -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ -// //This program is free software; you can redistribute it and/or modify //it under the terms of the GNU General Public License as published by //the Free Software Foundation; either version 2 of the License, or @@ -44,31 +40,32 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.Set; import java.util.TreeSet; import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; import net.yacy.cora.document.analysis.Classification; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.AnchorURL; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.util.ByteBuffer; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.retrieval.Request; import net.yacy.document.parser.html.ContentScraper; import net.yacy.document.parser.html.ImageEntry; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; public class Document { - private final DigestURI source; // the source url + private final DigestURL source; // the source url private final String mimeType; // mimeType as taken from http header private final String charset; // the charset of the document private final List keywords; // most resources provide a keyword field @@ -78,16 +75,16 @@ public class Document { private final List sections; // if present: more titles/headlines appearing in the document private final List descriptions; // an abstract, if present: short content description private Object text; // the clear text, all that is visible - private final Map anchors; // all links embedded as clickeable entities (anchor tags) - private final Map rss; // all embedded rss feeds - private final Map images; // all visible pictures in document + private final Collection anchors; // all links embedded as clickeable entities (anchor tags) + private final LinkedHashMap rss; // all embedded rss feeds + private final LinkedHashMap images; // all visible pictures in document // the anchors and images - Maps are URL-to-EntityDescription mappings. // The EntityDescription appear either as visible text in anchors or as alternative // text in image tags. - private Map audiolinks, videolinks, applinks, hyperlinks; - private Map inboundlinks, outboundlinks; + private LinkedHashMap audiolinks, videolinks, applinks, hyperlinks; + private LinkedHashMap inboundlinks, outboundlinks; private Map emaillinks; - private MultiProtocolURI favicon; + private MultiProtocolURL favicon; private boolean resorted; private final Set languages; private final boolean indexingDenied; @@ -96,7 +93,7 @@ public class Document { private final Map> generic_facets; // a map from vocabulary names to the set of tags for that vocabulary which apply for this document private final Date date; - public Document(final DigestURI location, final String mimeType, final String charset, + public Document(final DigestURL location, final String mimeType, final String charset, final Object parserObject, final Set languages, final String[] keywords, @@ -105,9 +102,9 @@ public Document(final DigestURI location, final String mimeType, final String ch final String[] sections, final List abstrcts, final double lon, final double lat, final Object text, - final Map anchors, - final Map rss, - final Map images, + final Collection anchors, + final LinkedHashMap rss, + final LinkedHashMap images, final boolean indexingDenied, final Date date) { this.source = location; @@ -129,9 +126,9 @@ public Document(final DigestURI location, final String mimeType, final String ch this.lon = 0.0d; this.lat = 0.0d; } - this.anchors = (anchors == null) ? new HashMap(0) : anchors; - this.rss = (rss == null) ? new HashMap(0) : rss; - this.images = (images == null) ? new HashMap() : images; + this.anchors = (anchors == null) ? new ArrayList(0) : anchors; + this.rss = (rss == null) ? new LinkedHashMap(0) : rss; + this.images = (images == null) ? new LinkedHashMap() : images; this.publisher = publisher; this.hyperlinks = null; this.audiolinks = null; @@ -308,7 +305,7 @@ public String dc_identifier() { return this.source.toNormalform(true); } - public DigestURI dc_source() { + public DigestURL dc_source() { return this.source; } @@ -405,13 +402,13 @@ public List getKeywords() { return this.keywords; } - public Map getAnchors() { + public Collection getAnchors() { // returns all links embedded as anchors (clickeable entities) // this is a url(String)/text(String) map return this.anchors; } - public Map getRSS() { + public Map getRSS() { // returns all links embedded as anchors (clickeable entities) // this is a url(String)/text(String) map return this.rss; @@ -420,30 +417,30 @@ public Map getRSS() { // the next three methods provide a calculated view on the getAnchors/getImages: - public Map getHyperlinks() { + public Map getHyperlinks() { // this is a subset of the getAnchor-set: only links to other hyperrefs if (!this.resorted) resortLinks(); return this.hyperlinks; } - public Map getAudiolinks() { + public Map getAudiolinks() { if (!this.resorted) resortLinks(); return this.audiolinks; } - public Map getVideolinks() { + public Map getVideolinks() { if (!this.resorted) resortLinks(); return this.videolinks; } - public Map getImages() { + public Map getImages() { // returns all links enbedded as pictures (visible in document) // this resturns a htmlFilterImageEntry collection if (!this.resorted) resortLinks(); return this.images; } - public Map getApplinks() { + public Map getApplinks() { if (!this.resorted) resortLinks(); return this.applinks; } @@ -471,27 +468,25 @@ private void resortLinks() { synchronized (this) { if (this.resorted) return; // extract hyperlinks, medialinks and emaillinks from anchorlinks - DigestURI url; String u; int extpos, qpos; String ext = null; final String thishost = this.source.getHost(); - this.inboundlinks = new HashMap(); - this.outboundlinks = new HashMap(); - this.hyperlinks = new HashMap(); - this.videolinks = new HashMap(); - this.audiolinks = new HashMap(); - this.applinks = new HashMap(); - this.emaillinks = new HashMap(); - final Map collectedImages = new HashMap(); // this is a set that is collected now and joined later to the imagelinks - for (final Map.Entry entry: collectedImages.entrySet()) { + this.inboundlinks = new LinkedHashMap(); + this.outboundlinks = new LinkedHashMap(); + this.hyperlinks = new LinkedHashMap(); + this.videolinks = new LinkedHashMap(); + this.audiolinks = new LinkedHashMap(); + this.applinks = new LinkedHashMap(); + this.emaillinks = new LinkedHashMap(); + final Map collectedImages = new HashMap(); // this is a set that is collected now and joined later to the imagelinks + for (final Map.Entry entry: collectedImages.entrySet()) { if (entry.getKey().getHost().equals(thishost)) this.inboundlinks.put(entry.getKey(), "image"); else this.outboundlinks.put(entry.getKey(), "image"); } - for (final Map.Entry entry: this.anchors.entrySet()) { - url = entry.getKey(); + for (final DigestURL url: this.anchors) { if (url == null) continue; - final boolean noindex = entry.getValue().getProperty("rel", "").toLowerCase().indexOf("noindex",0) >= 0; - final boolean nofollow = entry.getValue().getProperty("rel", "").toLowerCase().indexOf("nofollow",0) >= 0; + final boolean noindex = url.getProperties().getProperty("rel", "").toLowerCase().indexOf("noindex",0) >= 0; + final boolean nofollow = url.getProperties().getProperty("rel", "").toLowerCase().indexOf("nofollow",0) >= 0; if ((thishost == null && url.getHost() == null) || ((thishost != null && url.getHost() != null) && (url.getHost().endsWith(thishost) || @@ -501,7 +496,7 @@ private void resortLinks() { this.outboundlinks.put(url, "anchor" + (noindex ? " noindex" : "") + (nofollow ? " nofollow" : "")); } u = url.toNormalform(true); - final String name = entry.getValue().getProperty("name", ""); + final String name = url.getProperties().getProperty("name", ""); if (u.startsWith("mailto:")) { this.emaillinks.put(u.substring(7), name); } else { @@ -515,7 +510,7 @@ private void resortLinks() { if (Classification.isMediaExtension(ext)) { // this is not a normal anchor, its a media link if (Classification.isImageExtension(ext)) { - ContentScraper.addImage(collectedImages, new ImageEntry(url, name, -1, -1, -1)); + collectedImages.put(url, new ImageEntry(url, name, -1, -1, -1)); } else if (Classification.isAudioExtension(ext)) this.audiolinks.put(url, name); else if (Classification.isVideoExtension(ext)) this.videolinks.put(url, name); @@ -528,7 +523,7 @@ private void resortLinks() { } // add image links that we collected from the anchors to the image map - ContentScraper.addAllImages(this.images, collectedImages); + this.images.putAll(collectedImages); // expand the hyperlinks: // we add artificial hyperlinks to the hyperlink set @@ -550,21 +545,21 @@ private void resortLinks() { } } - public static Map allSubpaths(final Collection links) { + public static Map allSubpaths(final Collection links) { // links is either a Set of Strings (urls) or a Set of // htmlFilterImageEntries final Set h = new HashSet(); Iterator i = links.iterator(); Object o; - MultiProtocolURI url; + MultiProtocolURL url; String u; int pos; int l; while (i.hasNext()) try { o = i.next(); - if (o instanceof MultiProtocolURI) url = (MultiProtocolURI) o; - else if (o instanceof String) url = new MultiProtocolURI((String) o); + if (o instanceof MultiProtocolURL) url = (MultiProtocolURL) o; + else if (o instanceof String) url = new MultiProtocolURL((String) o); else if (o instanceof ImageEntry) url = ((ImageEntry) o).url(); else { assert false; @@ -585,11 +580,11 @@ public static Map allSubpaths(final Collection link } catch (final MalformedURLException e) { } // now convert the strings to yacyURLs i = h.iterator(); - final Map v = new HashMap(); + final Map v = new HashMap(); while (i.hasNext()) { u = (String) i.next(); try { - url = new MultiProtocolURI(u); + url = new MultiProtocolURL(u); v.put(url, "sub"); } catch (final MalformedURLException e) { } @@ -597,23 +592,23 @@ public static Map allSubpaths(final Collection link return v; } - private static Map allReflinks(final Collection links) { + private static Map allReflinks(final Collection links) { // links is either a Set of Strings (with urls) or // htmlFilterImageEntries // we find all links that are part of a reference inside a url - final Map v = new HashMap(); + final Map v = new HashMap(); final Iterator i = links.iterator(); Object o; - DigestURI url = null; + DigestURL url = null; String u; int pos; loop: while (i.hasNext()) try { o = i.next(); - if (o instanceof DigestURI) - url = (DigestURI) o; + if (o instanceof DigestURL) + url = (DigestURL) o; else if (o instanceof String) - url = new DigestURI((String) o); + url = new DigestURL((String) o); else if (o instanceof ImageEntry) url = ((ImageEntry) o).url(); else { @@ -627,7 +622,7 @@ else if (o instanceof ImageEntry) u = u.substring(pos); while ((pos = u.toLowerCase().indexOf("http://", 7)) > 0) u = u.substring(pos); - url = new DigestURI(u); + url = new DigestURL(u); if (!(v.containsKey(url))) v.put(url, "ref"); continue loop; @@ -637,7 +632,7 @@ else if (o instanceof ImageEntry) u = "http:/" + u.substring(pos); while ((pos = u.toLowerCase().indexOf("/www.", 7)) > 0) u = "http:/" + u.substring(pos); - url = new DigestURI(u); + url = new DigestURL(u); if (!(v.containsKey(url))) v.put(url, "ref"); continue loop; @@ -659,23 +654,23 @@ public void addSubDocuments(final Document[] docs) throws IOException { } FileUtils.copy(doc.getTextStream(), (ByteArrayOutputStream) this.text); - this.anchors.putAll(doc.getAnchors()); + this.anchors.addAll(doc.getAnchors()); this.rss.putAll(doc.getRSS()); - ContentScraper.addAllImages(this.images, doc.getImages()); + this.images.putAll(doc.getImages()); } } /** * @return the {@link URL} to the favicon that belongs to the document */ - public MultiProtocolURI getFavicon() { + public MultiProtocolURL getFavicon() { return this.favicon; } /** * @param faviconURL the {@link URL} to the favicon that belongs to the document */ - public void setFavicon(final MultiProtocolURI faviconURL) { + public void setFavicon(final MultiProtocolURL faviconURL) { this.favicon = faviconURL; } @@ -699,14 +694,14 @@ public int outboundLinkNofollowCount() { return c; } - public Set inboundLinks() { + public LinkedHashMap inboundLinks() { if (this.inboundlinks == null) resortLinks(); - return (this.inboundlinks == null) ? null : this.inboundlinks.keySet(); + return (this.inboundlinks == null) ? null : this.inboundlinks; } - public Set outboundLinks() { + public LinkedHashMap outboundLinks() { if (this.outboundlinks == null) resortLinks(); - return (this.outboundlinks == null) ? null : this.outboundlinks.keySet(); + return (this.outboundlinks == null) ? null : this.outboundlinks; } public boolean indexingDenied() { @@ -774,7 +769,7 @@ public synchronized void close() { * @param docs * @return */ - public static Document mergeDocuments(final DigestURI location, final String globalMime, final Document[] docs) { + public static Document mergeDocuments(final DigestURL location, final String globalMime, final Document[] docs) { if (docs == null || docs.length == 0) return null; if (docs.length == 1) return docs[0]; @@ -786,9 +781,9 @@ public static Document mergeDocuments(final DigestURI location, final String glo final List descriptions = new ArrayList(); final Collection titles = new LinkedHashSet(); final Collection sectionTitles = new LinkedHashSet(); - final Map anchors = new HashMap(); - final Map rss = new HashMap(); - final Map images = new HashMap(); + final List anchors = new ArrayList(); + final LinkedHashMap rss = new LinkedHashMap(); + final LinkedHashMap images = new LinkedHashMap(); double lon = 0.0d, lat = 0.0d; Date date = new Date(); @@ -825,9 +820,9 @@ public static Document mergeDocuments(final DigestURI location, final String glo ConcurrentLog.logException(e); } } - anchors.putAll(doc.getAnchors()); + anchors.addAll(doc.getAnchors()); rss.putAll(doc.getRSS()); - ContentScraper.addAllImages(images, doc.getImages()); + images.putAll(doc.getImages()); if (doc.lon() != 0.0 && doc.lat() != 0.0) { lon = doc.lon(); lat = doc.lat(); } if (doc.date.before(date)) date = doc.date; } @@ -865,24 +860,24 @@ public static Document mergeDocuments(final DigestURI location, final String glo date); } - public static Map getHyperlinks(final Document[] documents) { - final Map result = new HashMap(); + public static Map getHyperlinks(final Document[] documents) { + final Map result = new HashMap(); for (final Document d: documents) { result.putAll(d.getHyperlinks()); final Object parser = d.getParserObject(); if (parser instanceof ContentScraper) { final ContentScraper html = (ContentScraper) parser; String refresh = html.getRefreshPath(); - if (refresh != null && refresh.length() > 0) try {result.put(new DigestURI(refresh), "refresh");} catch (final MalformedURLException e) {} - DigestURI canonical = html.getCanonical(); + if (refresh != null && refresh.length() > 0) try {result.put(new DigestURL(refresh), "refresh");} catch (final MalformedURLException e) {} + DigestURL canonical = html.getCanonical(); if (canonical != null) result.put(canonical, "canonical"); } } return result; } - public static Map getImagelinks(final Document[] documents) { - final Map result = new HashMap(); + public static Map getImagelinks(final Document[] documents) { + final Map result = new HashMap(); for (final Document d: documents) { for (final ImageEntry imageReference : d.getImages().values()) { // construct a image name which contains the document title to enhance the search process for images @@ -892,30 +887,30 @@ public static Map getImagelinks(final Document[] documents) { return result; } - public static Map getAudiolinks(final Document[] documents) { - final Map result = new HashMap(); + public static Map getAudiolinks(final Document[] documents) { + final Map result = new HashMap(); for (final Document d: documents) { - for (Map.Entry e: d.audiolinks.entrySet()) { + for (Map.Entry e: d.audiolinks.entrySet()) { result.put(e.getKey(), description(d, e.getValue())); } } return result; } - public static Map getVideolinks(final Document[] documents) { - final Map result = new HashMap(); + public static Map getVideolinks(final Document[] documents) { + final Map result = new HashMap(); for (final Document d: documents) { - for (Map.Entry e: d.videolinks.entrySet()) { + for (Map.Entry e: d.videolinks.entrySet()) { result.put(e.getKey(), description(d, e.getValue())); } } return result; } - public static Map getApplinks(final Document[] documents) { - final Map result = new HashMap(); + public static Map getApplinks(final Document[] documents) { + final Map result = new HashMap(); for (final Document d: documents) { - for (Map.Entry e: d.applinks.entrySet()) { + for (Map.Entry e: d.applinks.entrySet()) { result.put(e.getKey(), description(d, e.getValue())); } } diff --git a/source/net/yacy/document/LibraryProvider.java b/source/net/yacy/document/LibraryProvider.java index 592e1ac0bc..70dfecfdcf 100644 --- a/source/net/yacy/document/LibraryProvider.java +++ b/source/net/yacy/document/LibraryProvider.java @@ -42,8 +42,8 @@ import java.util.zip.ZipException; import java.util.zip.ZipFile; -import net.yacy.cora.document.MultiProtocolURI; import net.yacy.cora.document.WordCache; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.geo.GeonamesLocation; import net.yacy.cora.geo.OpenGeoDBLocation; import net.yacy.cora.geo.OverarchingLocation; @@ -90,7 +90,7 @@ public static enum Dictionary { private Dictionary(final String nickname, final String url) { try { - this.filename = (new MultiProtocolURI(url)).getFileName(); + this.filename = (new MultiProtocolURL(url)).getFileName(); } catch (final MalformedURLException e ) { assert false; } diff --git a/source/net/yacy/document/Parser.java b/source/net/yacy/document/Parser.java index 74a80d1b60..f930dcfd5b 100644 --- a/source/net/yacy/document/Parser.java +++ b/source/net/yacy/document/Parser.java @@ -26,8 +26,8 @@ import java.io.InputStream; import java.util.Set; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; public interface Parser { @@ -54,7 +54,7 @@ public interface Parser { * @throws InterruptedException */ public Document[] parse( - DigestURI url, + DigestURL url, String mimeType, String charset, InputStream source @@ -91,22 +91,22 @@ public Document[] parse( public class Failure extends Exception { private static final long serialVersionUID = 2278214953869122883L; - private MultiProtocolURI url = null; + private MultiProtocolURL url = null; public Failure() { super(); } - public Failure(final String message, final MultiProtocolURI url) { + public Failure(final String message, final MultiProtocolURL url) { super(message + "; url = " + url.toNormalform(true)); this.url = url; } - public Failure(final String message, final MultiProtocolURI url, Throwable e) { + public Failure(final String message, final MultiProtocolURL url, Throwable e) { super(message + "; url = " + url.toNormalform(true), e); this.url = url; } - public MultiProtocolURI getURL() { + public MultiProtocolURL getURL() { return this.url; } } diff --git a/source/net/yacy/document/TextParser.java b/source/net/yacy/document/TextParser.java index 23fd860c4a..52ebf8f7b7 100644 --- a/source/net/yacy/document/TextParser.java +++ b/source/net/yacy/document/TextParser.java @@ -33,8 +33,9 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.document.parser.audioTagParser; import net.yacy.document.parser.bzipParser; import net.yacy.document.parser.csvParser; @@ -63,7 +64,6 @@ import net.yacy.document.parser.augment.AugmentParser; import net.yacy.document.parser.images.genericImageParser; import net.yacy.document.parser.rdfa.impl.RDFaParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.MemoryControl; import net.yacy.search.Switchboard; @@ -156,7 +156,7 @@ private static void initParser(final Parser parser) { } public static Document[] parseSource( - final DigestURI location, + final DigestURL location, final String mimeType, final String charset, final File sourceFile @@ -186,7 +186,7 @@ public static Document[] parseSource( } public static Document[] parseSource( - final DigestURI location, + final DigestURL location, String mimeType, final String charset, final byte[] content @@ -197,7 +197,7 @@ public static Document[] parseSource( try { idioms = parsers(location, mimeType); } catch (final Parser.Failure e) { - final String errorMsg = "Parser Failure for extension '" + MultiProtocolURI.getFileExtension(location.getFileName()) + "' or mimetype '" + mimeType + "': " + e.getMessage(); + final String errorMsg = "Parser Failure for extension '" + MultiProtocolURL.getFileExtension(location.getFileName()) + "' or mimetype '" + mimeType + "': " + e.getMessage(); AbstractParser.log.warn(errorMsg); throw new Parser.Failure(errorMsg, location); } @@ -209,7 +209,7 @@ public static Document[] parseSource( } public static Document[] parseSource( - final DigestURI location, + final DigestURL location, String mimeType, final String charset, final long contentLength, @@ -221,7 +221,7 @@ public static Document[] parseSource( try { idioms = parsers(location, mimeType); } catch (final Parser.Failure e) { - final String errorMsg = "Parser Failure for extension '" + MultiProtocolURI.getFileExtension(location.getFileName()) + "' or mimetype '" + mimeType + "': " + e.getMessage(); + final String errorMsg = "Parser Failure for extension '" + MultiProtocolURL.getFileExtension(location.getFileName()) + "' or mimetype '" + mimeType + "': " + e.getMessage(); AbstractParser.log.warn(errorMsg); throw new Parser.Failure(errorMsg, location); } @@ -248,14 +248,14 @@ public static Document[] parseSource( } private static Document[] parseSource( - final DigestURI location, + final DigestURL location, final String mimeType, final Parser parser, final String charset, final InputStream sourceStream ) throws Parser.Failure { if (AbstractParser.log.isFine()) AbstractParser.log.fine("Parsing '" + location + "' from stream"); - final String fileExt = MultiProtocolURI.getFileExtension(location.getFileName()); + final String fileExt = MultiProtocolURL.getFileExtension(location.getFileName()); final String documentCharset = htmlParser.patchCharsetEncoding(charset); assert parser != null; @@ -269,13 +269,13 @@ private static Document[] parseSource( } private static Document[] parseSource( - final DigestURI location, + final DigestURL location, final String mimeType, final Set parsers, final String charset, final byte[] sourceArray ) throws Parser.Failure { - final String fileExt = MultiProtocolURI.getFileExtension(location.getFileName()); + final String fileExt = MultiProtocolURL.getFileExtension(location.getFileName()); if (AbstractParser.log.isFine()) AbstractParser.log.fine("Parsing " + location + " with mimeType '" + mimeType + "' and file extension '" + fileExt + "' from byte[]"); final String documentCharset = htmlParser.patchCharsetEncoding(charset); assert !parsers.isEmpty(); @@ -337,7 +337,7 @@ private static Document[] parseSource( * @param mimeType * @return returns null if the content is supported. If the content is not supported, return a error string. */ - public static String supports(final MultiProtocolURI url, final String mimeType) { + public static String supports(final MultiProtocolURL url, final String mimeType) { try { // try to get a parser. If this works, we don't need the parser itself, we just return null to show that everything is ok. final Set idioms = parsers(url, mimeType); @@ -361,11 +361,11 @@ public static String supports(final MultiProtocolURI url, final String mimeType) * @return a list of Idiom parsers that may be appropriate for the given criteria * @throws Parser.Failure */ - private static Set parsers(final MultiProtocolURI url, String mimeType1) throws Parser.Failure { + private static Set parsers(final MultiProtocolURL url, String mimeType1) throws Parser.Failure { final Set idioms = new LinkedHashSet(2); // LinkedSet to maintain order (genericParser should be last) // check extension - String ext = MultiProtocolURI.getFileExtension(url.getFileName()); + String ext = MultiProtocolURL.getFileExtension(url.getFileName()); Set idiom; if (ext != null && ext.length() > 0) { ext = ext.toLowerCase(); @@ -430,12 +430,12 @@ public static String supportsExtension(final String ext) { * @param extention * @return an error if the extension is not supported, null otherwise */ - public static String supportsExtension(final MultiProtocolURI url) { - return supportsExtension(MultiProtocolURI.getFileExtension(url.getFileName()).toLowerCase()); + public static String supportsExtension(final MultiProtocolURL url) { + return supportsExtension(MultiProtocolURL.getFileExtension(url.getFileName()).toLowerCase()); } - public static String mimeOf(final MultiProtocolURI url) { - return mimeOf(MultiProtocolURI.getFileExtension(url.getFileName())); + public static String mimeOf(final MultiProtocolURL url) { + return mimeOf(MultiProtocolURL.getFileExtension(url.getFileName())); } public static String mimeOf(final String ext) { diff --git a/source/net/yacy/document/content/DCEntry.java b/source/net/yacy/document/content/DCEntry.java index adb822bb28..42d57d33fa 100644 --- a/source/net/yacy/document/content/DCEntry.java +++ b/source/net/yacy/document/content/DCEntry.java @@ -40,9 +40,9 @@ import org.apache.solr.common.params.MultiMapSolrParams; import net.yacy.cora.date.ISO8601Formatter; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.Document; -import net.yacy.kelondro.data.meta.DigestURI; public class DCEntry extends MultiMapSolrParams { @@ -61,7 +61,7 @@ public DCEntry() { } public DCEntry( - DigestURI url, + DigestURL url, Date date, String title, String author, @@ -115,7 +115,7 @@ public Date getDate() { } } - public DigestURI getIdentifier(boolean useRelationAsAlternative) { + public DigestURL getIdentifier(boolean useRelationAsAlternative) { String u = this.get("url"); if (u == null) u = this.get("dc:identifier"); if (u == null) return useRelationAsAlternative ? getRelation() : null; @@ -125,10 +125,10 @@ public DigestURI getIdentifier(boolean useRelationAsAlternative) { u = bestU(urls); } try { - return new DigestURI(u); + return new DigestURL(u); } catch (final MalformedURLException e) { if (useRelationAsAlternative) { - DigestURI relation = this.getRelation(); + DigestURL relation = this.getRelation(); if (relation != null) return relation; ConcurrentLog.warn("DCEntry", "getIdentifier: url is bad, relation also: " + e.getMessage()); } @@ -137,7 +137,7 @@ public DigestURI getIdentifier(boolean useRelationAsAlternative) { } } - public DigestURI getRelation() { + public DigestURL getRelation() { String u = this.get("dc:relation"); if (u == null) return null; String[] urls = u.split(";"); @@ -146,7 +146,7 @@ public DigestURI getRelation() { u = bestU(urls); } try { - return new DigestURI(u); + return new DigestURL(u); } catch (final MalformedURLException e) { ConcurrentLog.warn("DCEntry", "getRelation: url is bad: " + e.getMessage()); return null; diff --git a/source/net/yacy/document/content/dao/ImportDump.java b/source/net/yacy/document/content/dao/ImportDump.java index ff5a5d58b3..1ae468a612 100644 --- a/source/net/yacy/document/content/dao/ImportDump.java +++ b/source/net/yacy/document/content/dao/ImportDump.java @@ -30,7 +30,7 @@ import java.sql.SQLException; import java.sql.Statement; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.util.ConcurrentLog; import net.yacy.kelondro.util.FileUtils; diff --git a/source/net/yacy/document/content/dao/PhpBB3Dao.java b/source/net/yacy/document/content/dao/PhpBB3Dao.java index 412b13ba82..d443ceac69 100644 --- a/source/net/yacy/document/content/dao/PhpBB3Dao.java +++ b/source/net/yacy/document/content/dao/PhpBB3Dao.java @@ -39,10 +39,10 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.content.DCEntry; import net.yacy.document.content.SurrogateReader; -import net.yacy.kelondro.data.meta.DigestURI; public class PhpBB3Dao implements Dao { @@ -208,9 +208,9 @@ public void run() { } protected DCEntry parseResultSet(ResultSet rs) throws SQLException, MalformedURLException { - DigestURI url; + DigestURL url; int item = rs.getInt("post_id"); - url = new DigestURI(this.urlstub + "/viewtopic.php?t=" + item); + url = new DigestURL(this.urlstub + "/viewtopic.php?t=" + item); String subject = rs.getString("post_subject"); String text = xmlCleaner(rs.getString("post_text")); String user = getUser(rs.getInt("poster_id")); @@ -271,7 +271,7 @@ public int writeSurrogates( ) { try { // generate output file name and attributes - String targethost = new DigestURI(this.urlstub).getHost(); + String targethost = new DigestURL(this.urlstub).getHost(); int fc = 0; File outputfiletmp = null, outputfile = null; diff --git a/source/net/yacy/document/importer/MediawikiImporter.java b/source/net/yacy/document/importer/MediawikiImporter.java index 89ce403079..b6fa2f4453 100644 --- a/source/net/yacy/document/importer/MediawikiImporter.java +++ b/source/net/yacy/document/importer/MediawikiImporter.java @@ -50,7 +50,8 @@ import java.util.concurrent.TimeoutException; import java.util.zip.GZIPInputStream; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ByteBuffer; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.NumberTools; @@ -60,7 +61,6 @@ import net.yacy.document.Parser; import net.yacy.document.TextParser; import net.yacy.document.content.SurrogateReader; -import net.yacy.kelondro.data.meta.DigestURI; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; @@ -193,7 +193,7 @@ public void run() { q = this.urlStub.lastIndexOf('/'); if (q > 0) this.urlStub = this.urlStub.substring(0, q + 1); } - final DigestURI uri = new DigestURI(this.urlStub); + final DigestURL uri = new DigestURL(this.urlStub); this.hostport = uri.getHost(); if (uri.getPort() != 80) this.hostport += ":" + uri.getPort(); continue; @@ -501,7 +501,7 @@ public wikiparserrecord newRecord(final String hostport, final String urlStub, f public class wikiparserrecord { public String title; String source, html, hostport, urlStub; - DigestURI url; + DigestURL url; Document document; public wikiparserrecord(final String hostport, final String urlStub, final String title, final StringBuilder sb) { this.title = title; @@ -520,7 +520,7 @@ public void genHTML() throws IOException { } public void genDocument() throws Parser.Failure { try { - this.url = new DigestURI(this.urlStub + this.title); + this.url = new DigestURL(this.urlStub + this.title); final Document[] parsed = TextParser.parseSource(this.url, "text/html", "UTF-8", UTF8.getBytes(this.html)); this.document = Document.mergeDocuments(this.url, "text/html", parsed); // the wiki parser is not able to find the proper title in the source text, so it must be set here diff --git a/source/net/yacy/document/importer/OAIListFriendsLoader.java b/source/net/yacy/document/importer/OAIListFriendsLoader.java index 13ad764bc8..d492f90652 100644 --- a/source/net/yacy/document/importer/OAIListFriendsLoader.java +++ b/source/net/yacy/document/importer/OAIListFriendsLoader.java @@ -39,12 +39,12 @@ import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.retrieval.Response; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.LoaderDispatcher; @@ -63,7 +63,7 @@ public static void init(final LoaderDispatcher loader, final Map m listFriends.putAll(moreFriends); if (loader != null) for (final Map.Entry oaiFriend: listFriends.entrySet()) { try { - loader.loadIfNotExistBackground(new DigestURI(oaiFriend.getKey()), oaiFriend.getValue(), Integer.MAX_VALUE, null, agent); + loader.loadIfNotExistBackground(new DigestURL(oaiFriend.getKey()), oaiFriend.getValue(), Integer.MAX_VALUE, null, agent); } catch (final MalformedURLException e) { } } @@ -88,7 +88,7 @@ public static Map getListFriends(final LoaderDispatcher loader, Map m; for (final Map.Entry oaiFriend: listFriends.entrySet()) try { if (!oaiFriend.getValue().exists()) { - final Response response = loader == null ? null : loader.load(loader.request(new DigestURI(oaiFriend.getKey()), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, agent); + final Response response = loader == null ? null : loader.load(loader.request(new DigestURL(oaiFriend.getKey()), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, agent); if (response != null) FileUtils.copy(response.getContent(), oaiFriend.getValue()); } diff --git a/source/net/yacy/document/importer/OAIPMHImporter.java b/source/net/yacy/document/importer/OAIPMHImporter.java index ae55b51ac9..222ea279e3 100644 --- a/source/net/yacy/document/importer/OAIPMHImporter.java +++ b/source/net/yacy/document/importer/OAIPMHImporter.java @@ -33,9 +33,9 @@ import java.util.concurrent.ConcurrentHashMap; import net.yacy.cora.date.GenericFormatter; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.repository.LoaderDispatcher; import net.yacy.search.Switchboard; @@ -53,7 +53,7 @@ public class OAIPMHImporter extends Thread implements Importer, Comparable finishedJobs = new ConcurrentHashMap(); private final LoaderDispatcher loader; - private DigestURI source; + private DigestURL source; private int recordsCount, chunkCount, completeListSize; private final long startTime; private long finishTime; @@ -62,7 +62,7 @@ public class OAIPMHImporter extends Thread implements Importer, Comparable getLoadedOAIServer(File surrogates) { * @param source * @return a string that is a key for the given host */ - public static final String hostID(DigestURI source) { + public static final String hostID(DigestURL source) { String s = ResumptionToken.truncatedURL(source); if (s.endsWith("?")) s = s.substring(0, s.length() - 1); if (s.endsWith("/")) s = s.substring(0, s.length() - 1); @@ -253,7 +253,7 @@ public static final String hostID(DigestURI source) { * @param source * @return a file name for the given source. It will be different for each call for same hosts because it contains a date stamp */ - public static final String filename4Source(DigestURI source) { + public static final String filename4Source(DigestURL source) { return filenamePrefix + OAIPMHImporter.filenameSeparationChar + OAIPMHImporter.hostID(source) + OAIPMHImporter.filenameSeparationChar + GenericFormatter.SHORT_MILSEC_FORMATTER.format() + ".xml"; diff --git a/source/net/yacy/document/importer/OAIPMHLoader.java b/source/net/yacy/document/importer/OAIPMHLoader.java index 6aa60006e8..fdda7dd307 100644 --- a/source/net/yacy/document/importer/OAIPMHLoader.java +++ b/source/net/yacy/document/importer/OAIPMHLoader.java @@ -25,11 +25,11 @@ import java.io.File; import java.io.IOException; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.retrieval.Response; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.LoaderDispatcher; @@ -42,10 +42,10 @@ public class OAIPMHLoader { - private final DigestURI source; + private final DigestURL source; private final ResumptionToken resumptionToken; - public OAIPMHLoader(final LoaderDispatcher loader, final DigestURI source, final File targetDir, final ClientIdentification.Agent agent) throws IOException { + public OAIPMHLoader(final LoaderDispatcher loader, final DigestURL source, final File targetDir, final ClientIdentification.Agent agent) throws IOException { this.source = source; // load the file from the net diff --git a/source/net/yacy/document/importer/ResumptionToken.java b/source/net/yacy/document/importer/ResumptionToken.java index 0bb97d0c86..4fa1a472a1 100644 --- a/source/net/yacy/document/importer/ResumptionToken.java +++ b/source/net/yacy/document/importer/ResumptionToken.java @@ -36,9 +36,9 @@ import javax.xml.parsers.SAXParserFactory; import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.data.meta.DigestURI; import org.xml.sax.Attributes; import org.xml.sax.SAXException; @@ -57,9 +57,9 @@ public class ResumptionToken extends TreeMap { int recordCounter; - private final DigestURI source; + private final DigestURL source; - public ResumptionToken(final DigestURI source, final byte[] b) throws IOException { + public ResumptionToken(final DigestURL source, final byte[] b) throws IOException { super((Collator) insensitiveCollator.clone()); this.source = source; this.recordCounter = 0; @@ -105,7 +105,7 @@ public ResumptionToken( * @param url * @return a string containing the url up to and including the '?' */ - public static String truncatedURL(final DigestURI url) { + public static String truncatedURL(final DigestURL url) { String u = url.toNormalform(true); final int i = u.indexOf('?'); if (i > 0) u = u.substring(0, i + 1); @@ -126,7 +126,7 @@ public int getRecordCounter() { * @return * @throws IOException in case that no follow-up url can be generated; i.e. if the expiration date is exceeded */ - public DigestURI resumptionURL() throws IOException { + public DigestURL resumptionURL() throws IOException { // decide which kind of encoding strategy was used to get a resumptionToken: final String token = getToken(); @@ -136,7 +136,7 @@ public DigestURI resumptionURL() throws IOException { // encoded state if (token.indexOf("from=",0) >= 0) { - return new DigestURI(url + "verb=ListRecords&" + token); + return new DigestURL(url + "verb=ListRecords&" + token); } // cached result set @@ -147,7 +147,7 @@ public DigestURI resumptionURL() throws IOException { // the resumption token is still fresh } final String u = url + "verb=ListRecords&resumptionToken=" + escape(token); - return new DigestURI(u); + return new DigestURL(u); } public static StringBuilder escape(final String s) { diff --git a/source/net/yacy/document/parser/audioTagParser.java b/source/net/yacy/document/parser/audioTagParser.java index 82044c2cc1..e405996d02 100644 --- a/source/net/yacy/document/parser/audioTagParser.java +++ b/source/net/yacy/document/parser/audioTagParser.java @@ -35,12 +35,12 @@ import java.util.List; import java.util.Set; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; import org.jaudiotagger.audio.AudioFile; import org.jaudiotagger.audio.AudioFileIO; @@ -69,13 +69,13 @@ public audioTagParser() { } @Override - public Document[] parse(final DigestURI location, final String mimeType, + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { String filename = location.getFileName(); - final String fileext = '.' + MultiProtocolURI.getFileExtension(filename); - filename = filename.isEmpty() ? location.toTokens() : MultiProtocolURI.unescape(filename); + final String fileext = '.' + MultiProtocolURL.getFileExtension(filename); + filename = filename.isEmpty() ? location.toTokens() : MultiProtocolURL.unescape(filename); String mime = mimeType; // fix mimeType diff --git a/source/net/yacy/document/parser/augment/AugmentParser.java b/source/net/yacy/document/parser/augment/AugmentParser.java index 0d4bff25f5..8031696174 100644 --- a/source/net/yacy/document/parser/augment/AugmentParser.java +++ b/source/net/yacy/document/parser/augment/AugmentParser.java @@ -6,13 +6,13 @@ import java.util.Iterator; import java.util.Set; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.ymark.YMarkUtil; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.parser.rdfa.impl.RDFaParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; @@ -35,7 +35,7 @@ public AugmentParser() { } @Override - public Document[] parse(DigestURI url, String mimeType, String charset, InputStream source) throws Parser.Failure, InterruptedException { + public Document[] parse(DigestURL url, String mimeType, String charset, InputStream source) throws Parser.Failure, InterruptedException { Document[] htmlDocs = this.rdfaParser.parse(url, mimeType, charset, source); @@ -58,7 +58,7 @@ private void analyze(Document origDoc, DigestURI url, } } */ - private void parseAndAugment(Document origDoc, DigestURI url, @SuppressWarnings("unused") String mimeType, @SuppressWarnings("unused") String charset) { + private void parseAndAugment(Document origDoc, DigestURL url, @SuppressWarnings("unused") String mimeType, @SuppressWarnings("unused") String charset) { Iterator it; try { diff --git a/source/net/yacy/document/parser/bzipParser.java b/source/net/yacy/document/parser/bzipParser.java index 3f33f974fc..4e2f0ab7b2 100644 --- a/source/net/yacy/document/parser/bzipParser.java +++ b/source/net/yacy/document/parser/bzipParser.java @@ -31,11 +31,11 @@ import java.io.FileOutputStream; import java.io.InputStream; +import net.yacy.cora.document.id.DigestURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.TextParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; @@ -56,7 +56,7 @@ public bzipParser() { } @Override - public Document[] parse(final DigestURI location, final String mimeType, + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { diff --git a/source/net/yacy/document/parser/csvParser.java b/source/net/yacy/document/parser/csvParser.java index 6d50cbc935..b690485d92 100644 --- a/source/net/yacy/document/parser/csvParser.java +++ b/source/net/yacy/document/parser/csvParser.java @@ -33,10 +33,10 @@ import java.util.Date; import java.util.List; +import net.yacy.cora.document.id.DigestURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; /** * a parser for comma-separated values @@ -51,7 +51,7 @@ public csvParser() { } @Override - public Document[] parse(DigestURI location, String mimeType, String charset, InputStream source) throws Parser.Failure, InterruptedException { + public Document[] parse(DigestURL location, String mimeType, String charset, InputStream source) throws Parser.Failure, InterruptedException { // construct a document using all cells of the document // the first row is used as headline // all lines are artificially terminated by a '.' to separate them as sentence for the condenser. diff --git a/source/net/yacy/document/parser/docParser.java b/source/net/yacy/document/parser/docParser.java index 3a402a1c0e..297adf3f23 100644 --- a/source/net/yacy/document/parser/docParser.java +++ b/source/net/yacy/document/parser/docParser.java @@ -30,10 +30,10 @@ import java.io.InputStream; import java.util.Date; +import net.yacy.cora.document.id.DigestURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; import org.apache.poi.hwpf.extractor.WordExtractor; @@ -54,7 +54,7 @@ public docParser() { } @Override - public Document[] parse(final DigestURI location, final String mimeType, + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { diff --git a/source/net/yacy/document/parser/dwgParser.java b/source/net/yacy/document/parser/dwgParser.java index 6289b1a57d..7dff5137ed 100644 --- a/source/net/yacy/document/parser/dwgParser.java +++ b/source/net/yacy/document/parser/dwgParser.java @@ -25,10 +25,10 @@ import java.io.InputStream; +import net.yacy.cora.document.id.DigestURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.MemoryControl; import org.apache.poi.util.StringUtil; @@ -60,7 +60,7 @@ public dwgParser() { } @Override - public Document[] parse(final DigestURI location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { // check memory for parser if (!MemoryControl.request(200 * 1024 * 1024, true)) diff --git a/source/net/yacy/document/parser/genericParser.java b/source/net/yacy/document/parser/genericParser.java index 8fa1d85ff3..e285f06242 100644 --- a/source/net/yacy/document/parser/genericParser.java +++ b/source/net/yacy/document/parser/genericParser.java @@ -27,11 +27,11 @@ import java.io.InputStream; import java.util.Date; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; /** * this parser can parse just anything because it uses only the uri/file/path information @@ -45,7 +45,7 @@ public genericParser() { } @Override - public Document[] parse(final DigestURI location, final String mimeType, + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source1) throws Parser.Failure, InterruptedException { String filename = location.getFileName(); @@ -56,7 +56,7 @@ public Document[] parse(final DigestURI location, final String mimeType, this, null, null, - singleList(filename.isEmpty() ? location.toTokens() : MultiProtocolURI.unescape(filename)), // title + singleList(filename.isEmpty() ? location.toTokens() : MultiProtocolURL.unescape(filename)), // title "", // author location.getHost(), null, diff --git a/source/net/yacy/document/parser/gzipParser.java b/source/net/yacy/document/parser/gzipParser.java index 287ea5108e..72d9c952dd 100644 --- a/source/net/yacy/document/parser/gzipParser.java +++ b/source/net/yacy/document/parser/gzipParser.java @@ -32,11 +32,11 @@ import java.io.InputStream; import java.util.zip.GZIPInputStream; +import net.yacy.cora.document.id.DigestURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.TextParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; @@ -55,7 +55,7 @@ public gzipParser() { } @Override - public Document[] parse(final DigestURI location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { File tempFile = null; Document[] docs = null; diff --git a/source/net/yacy/document/parser/html/ContentScraper.java b/source/net/yacy/document/parser/html/ContentScraper.java index 1e9dd363c3..7b36537faa 100644 --- a/source/net/yacy/document/parser/html/ContentScraper.java +++ b/source/net/yacy/document/parser/html/ContentScraper.java @@ -36,10 +36,10 @@ import java.util.Date; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.regex.Matcher; @@ -48,7 +48,9 @@ import javax.swing.event.EventListenerList; import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.AnchorURL; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.sorting.ClusteredScoreMap; import net.yacy.cora.storage.SizeLimitedMap; import net.yacy.cora.storage.SizeLimitedSet; @@ -57,7 +59,6 @@ import net.yacy.document.SentenceReader; import net.yacy.document.parser.htmlParser; import net.yacy.document.parser.html.Evaluation.Element; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.io.CharBuffer; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.ISO639; @@ -125,13 +126,13 @@ private Tag(final TagType type) { } // class variables: collectors for links - private final Map anchors; - private final Map rss, css; - private final Set script, frames, iframes; - private final Map embeds; // urlhash/embed relation - private final Map images; // urlhash/image relation + private final List anchors; + private final LinkedHashMap rss, css; + private final LinkedHashMap embeds; // urlhash/embed relation + private final List images; + private final Set script, frames, iframes; private final Map metas; - private final Map hreflang, navigation; + private final Map hreflang, navigation; private LinkedHashSet titles; //private String headline; private List[] headlines; @@ -140,20 +141,20 @@ private Tag(final TagType type) { private final CharBuffer content; private final EventListenerList htmlFilterEventListeners; private double lon, lat; - private DigestURI canonical, publisher; + private DigestURL canonical, publisher; private final int maxLinks; private int breadcrumbs; /** - * {@link MultiProtocolURI} to the favicon that belongs to the document + * {@link MultiProtocolURL} to the favicon that belongs to the document */ - private MultiProtocolURI favicon; + private MultiProtocolURL favicon; /** - * The document root {@link MultiProtocolURI} + * The document root {@link MultiProtocolURL} */ - private DigestURI root; + private DigestURL root; /** * evaluation scores: count appearance of specific attributes @@ -161,7 +162,7 @@ private Tag(final TagType type) { private final Evaluation evaluationScores; @SuppressWarnings("unchecked") - public ContentScraper(final DigestURI root, int maxLinks) { + public ContentScraper(final DigestURL root, int maxLinks) { // the root value here will not be used to load the resource. // it is only the reference for relative links super(linkTags0, linkTags1); @@ -169,17 +170,17 @@ public ContentScraper(final DigestURI root, int maxLinks) { this.root = root; this.maxLinks = maxLinks; this.evaluationScores = new Evaluation(); - this.rss = new SizeLimitedMap(maxLinks); - this.css = new SizeLimitedMap(maxLinks); - this.anchors = new SizeLimitedMap(maxLinks); - this.images = new SizeLimitedMap(maxLinks); - this.embeds = new SizeLimitedMap(maxLinks); - this.frames = new SizeLimitedSet(maxLinks); - this.iframes = new SizeLimitedSet(maxLinks); + this.rss = new SizeLimitedMap(maxLinks); + this.css = new SizeLimitedMap(maxLinks); + this.anchors = new ArrayList(); + this.images = new ArrayList(); + this.embeds = new SizeLimitedMap(maxLinks); + this.frames = new SizeLimitedSet(maxLinks); + this.iframes = new SizeLimitedSet(maxLinks); this.metas = new SizeLimitedMap(maxLinks); - this.hreflang = new SizeLimitedMap(maxLinks); - this.navigation = new SizeLimitedMap(maxLinks); - this.script = new SizeLimitedSet(maxLinks); + this.hreflang = new SizeLimitedMap(maxLinks); + this.navigation = new SizeLimitedMap(maxLinks); + this.script = new SizeLimitedSet(maxLinks); this.titles = new LinkedHashSet(); this.headlines = new ArrayList[6]; for (int i = 0; i < this.headlines.length; i++) this.headlines[i] = new ArrayList(); @@ -202,19 +203,6 @@ public void finish() { this.content.trimToSize(); } - private void mergeAnchors(final DigestURI url, final Properties p) { - final Properties p0 = this.anchors.get(url); - if (p0 == null) { - this.anchors.put(url, p); - return; - } - // merge properties - for (final Entry entry: p.entrySet()) { - if (entry.getValue() != null && entry.getValue().toString().length() > 0) p0.put(entry.getKey(), entry.getValue()); - } - this.anchors.put(url, p0); - } - @Override public void scrapeText(final char[] newtext, final String insideTag) { // System.out.println("SCRAPE: " + UTF8.String(newtext)); @@ -290,7 +278,6 @@ public void scrapeText(final char[] newtext, final String insideTag) { // find http links inside text s = 0; String u; - DigestURI url; while (s < b.length()) { p = find(b, dpssp, s); if (p == Integer.MAX_VALUE) break; @@ -302,8 +289,7 @@ public void scrapeText(final char[] newtext, final String insideTag) { if (u.endsWith(".")) u = u.substring(0, u.length() - 1); // remove the '.' that was appended above s = p + 6; try { - url = new DigestURI(u); - mergeAnchors(url, new Properties()); + this.anchors.add(new AnchorURL(u)); continue; } catch (final MalformedURLException e) {} } @@ -325,9 +311,9 @@ private static final int find(final String s, final Pattern m, final int start) return (p < 0) ? Integer.MAX_VALUE : p; } - private DigestURI absolutePath(final String relativePath) { + private AnchorURL absolutePath(final String relativePath) { try { - return DigestURI.newURL(this.root, relativePath); + return AnchorURL.newAnchor(this.root, relativePath); } catch (final Exception e) { return null; } @@ -339,24 +325,25 @@ public void scrapeTag0(final String tagname, final Properties tagopts) { final String src = tagopts.getProperty("src", EMPTY_STRING); try { if (src.length() > 0) { - final DigestURI url = absolutePath(src); + final DigestURL url = absolutePath(src); if (url != null) { final int width = Integer.parseInt(tagopts.getProperty("width", "-1")); final int height = Integer.parseInt(tagopts.getProperty("height", "-1")); final ImageEntry ie = new ImageEntry(url, tagopts.getProperty("alt", EMPTY_STRING), width, height, -1); - addImage(this.images, ie); + this.images.add(ie); } } } catch (final NumberFormatException e) {} this.evaluationScores.match(Element.imgpath, src); } else if(tagname.equalsIgnoreCase("base")) { try { - this.root = new DigestURI(tagopts.getProperty("href", EMPTY_STRING)); + this.root = new DigestURL(tagopts.getProperty("href", EMPTY_STRING)); } catch (final MalformedURLException e) {} } else if (tagname.equalsIgnoreCase("frame")) { - final DigestURI src = absolutePath(tagopts.getProperty("src", EMPTY_STRING)); + final AnchorURL src = absolutePath(tagopts.getProperty("src", EMPTY_STRING)); tagopts.put("src", src.toNormalform(true)); - mergeAnchors(src, tagopts /* with property "name" */); + src.getProperties().putAll(tagopts); + this.anchors.add(src); this.frames.add(src); this.evaluationScores.match(Element.framepath, src.toNormalform(true)); } else if (tagname.equalsIgnoreCase("body")) { @@ -392,13 +379,14 @@ public void scrapeTag0(final String tagname, final Properties tagopts) { final String href = tagopts.getProperty("href", EMPTY_STRING); if (href.length() > 0) { tagopts.put("nme", areatitle); - DigestURI url = absolutePath(href); + AnchorURL url = absolutePath(href); tagopts.put("href", url.toNormalform(true)); - mergeAnchors(url, tagopts); + url.getProperties().putAll(tagopts); + this.anchors.add(url); } } else if (tagname.equalsIgnoreCase("link")) { final String href = tagopts.getProperty("href", EMPTY_STRING); - final DigestURI newLink = absolutePath(href); + final AnchorURL newLink = absolutePath(href); if (newLink != null) { tagopts.put("href", newLink.toNormalform(true)); @@ -409,11 +397,12 @@ public void scrapeTag0(final String tagname, final Properties tagopts) { if (rel.equalsIgnoreCase("shortcut icon")) { final ImageEntry ie = new ImageEntry(newLink, linktitle, -1, -1, -1); - this.images.put(ie.url(), ie); + this.images.add(ie); this.favicon = newLink; } else if (rel.equalsIgnoreCase("canonical")) { tagopts.put("name", this.titles.size() == 0 ? "" : this.titles.iterator().next()); - mergeAnchors(newLink, tagopts); + newLink.getProperties().putAll(tagopts); + this.anchors.add(newLink); this.canonical = newLink; } else if (rel.equalsIgnoreCase("publisher")) { this.publisher = newLink; @@ -428,35 +417,39 @@ public void scrapeTag0(final String tagname, final Properties tagopts) { this.evaluationScores.match(Element.csspath, href); } else if (!rel.equalsIgnoreCase("stylesheet") && !rel.equalsIgnoreCase("alternate stylesheet")) { tagopts.put("name", linktitle); - mergeAnchors(newLink, tagopts); + newLink.getProperties().putAll(tagopts); + this.anchors.add(newLink); } } } else if(tagname.equalsIgnoreCase("embed")) { final String src = tagopts.getProperty("src", EMPTY_STRING); try { if (src.length() > 0) { - final DigestURI url = absolutePath(src); + final AnchorURL url = absolutePath(src); if (url != null) { final int width = Integer.parseInt(tagopts.getProperty("width", "-1")); final int height = Integer.parseInt(tagopts.getProperty("height", "-1")); tagopts.put("src", url.toNormalform(true)); final EmbedEntry ie = new EmbedEntry(url, width, height, tagopts.getProperty("type", EMPTY_STRING), tagopts.getProperty("pluginspage", EMPTY_STRING)); this.embeds.put(url, ie); - mergeAnchors(url, tagopts); + url.getProperties().putAll(tagopts); + this.anchors.add(url); } } } catch (final NumberFormatException e) {} } else if(tagname.equalsIgnoreCase("param")) { final String name = tagopts.getProperty("name", EMPTY_STRING); if (name.equalsIgnoreCase("movie")) { - DigestURI url = absolutePath(tagopts.getProperty("value", EMPTY_STRING)); + AnchorURL url = absolutePath(tagopts.getProperty("value", EMPTY_STRING)); tagopts.put("value", url.toNormalform(true)); - mergeAnchors(url, tagopts /* with property "name" */); + url.getProperties().putAll(tagopts); + this.anchors.add(url); } } else if (tagname.equalsIgnoreCase("iframe")) { - final DigestURI src = absolutePath(tagopts.getProperty("src", EMPTY_STRING)); + final AnchorURL src = absolutePath(tagopts.getProperty("src", EMPTY_STRING)); tagopts.put("src", src.toNormalform(true)); - mergeAnchors(src, tagopts /* with property "name" */); + src.getProperties().putAll(tagopts); + this.anchors.add(src); this.iframes.add(src); this.evaluationScores.match(Element.iframepath, src.toNormalform(true)); } else if (tagname.equalsIgnoreCase("html")) { @@ -474,58 +467,59 @@ public void scrapeTag1(final String tagname, final Properties tagopts, char[] te // System.out.println("ScrapeTag1: tagname=" + tagname + ", opts=" + tagopts.toString() + ", text=" + UTF8.String(text)); if (tagname.equalsIgnoreCase("a") && text.length < 2048) { final String href = tagopts.getProperty("href", EMPTY_STRING); - DigestURI url; + AnchorURL url; if ((href.length() > 0) && ((url = absolutePath(href)) != null)) { - final String ext = MultiProtocolURI.getFileExtension(url.getFileName()); + final String ext = MultiProtocolURL.getFileExtension(url.getFileName()); if (ext.equals("png") || ext.equals("gif") || ext.equals("jpg") || ext.equals("jpeg") || ext.equals("tiff") || ext.equals("tif")) { // special handling of such urls: put them to the image urls - final ImageEntry ie = new ImageEntry(url, recursiveParse(text), -1, -1, -1); - addImage(this.images, ie); + final ImageEntry ie = new ImageEntry(url, recursiveParse(url, text), -1, -1, -1); + this.images.add(ie); } else { - tagopts.put("text", recursiveParse(text)); + tagopts.put("text", recursiveParse(url, text)); tagopts.put("href", url.toNormalform(true)); // we must assign this because the url may have resolved backpaths and may not be absolute - mergeAnchors(url, tagopts); + url.getProperties().putAll(tagopts); + this.anchors.add(url); } } this.evaluationScores.match(Element.apath, href); } final String h; if ((tagname.equalsIgnoreCase("h1")) && (text.length < 1024)) { - h = recursiveParse(text); + h = recursiveParse(null, text); if (h.length() > 0) this.headlines[0].add(h); } else if((tagname.equalsIgnoreCase("h2")) && (text.length < 1024)) { - h = recursiveParse(text); + h = recursiveParse(null, text); if (h.length() > 0) this.headlines[1].add(h); } else if ((tagname.equalsIgnoreCase("h3")) && (text.length < 1024)) { - h = recursiveParse(text); + h = recursiveParse(null, text); if (h.length() > 0) this.headlines[2].add(h); } else if ((tagname.equalsIgnoreCase("h4")) && (text.length < 1024)) { - h = recursiveParse(text); + h = recursiveParse(null, text); if (h.length() > 0) this.headlines[3].add(h); } else if ((tagname.equalsIgnoreCase("h5")) && (text.length < 1024)) { - h = recursiveParse(text); + h = recursiveParse(null, text); if (h.length() > 0) this.headlines[4].add(h); } else if ((tagname.equalsIgnoreCase("h6")) && (text.length < 1024)) { - h = recursiveParse(text); + h = recursiveParse(null, text); if (h.length() > 0) this.headlines[5].add(h); } else if ((tagname.equalsIgnoreCase("title")) && (text.length < 1024)) { - String t = recursiveParse(text); + String t = recursiveParse(null, text); this.titles.add(t); this.evaluationScores.match(Element.title, t); } else if ((tagname.equalsIgnoreCase("b")) && (text.length < 1024)) { - h = recursiveParse(text); + h = recursiveParse(null, text); if (h.length() > 0) this.bold.inc(h); } else if ((tagname.equalsIgnoreCase("strong")) && (text.length < 1024)) { - h = recursiveParse(text); + h = recursiveParse(null, text); if (h.length() > 0) this.bold.inc(h); } else if ((tagname.equalsIgnoreCase("i")) && (text.length < 1024)) { - h = recursiveParse(text); + h = recursiveParse(null, text); if (h.length() > 0) this.italic.inc(h); } else if ((tagname.equalsIgnoreCase("u")) && (text.length < 1024)) { - h = recursiveParse(text); + h = recursiveParse(null, text); if (h.length() > 0) this.underline.inc(h); } else if ((tagname.equalsIgnoreCase("li")) && (text.length < 1024)) { - h = recursiveParse(text); + h = recursiveParse(null, text); if (h.length() > 0) this.li.add(h); } else if (tagname.equalsIgnoreCase("script")) { final String src = tagopts.getProperty("src", EMPTY_STRING); @@ -547,7 +541,7 @@ public void scrapeComment(final char[] comment) { this.evaluationScores.match(Element.comment, LB.matcher(new String(comment)).replaceAll(" ")); } - private String recursiveParse(final char[] inlineHtml) { + private String recursiveParse(final DigestURL linkurl, final char[] inlineHtml) { if (inlineHtml.length < 14) return cleanLine(CharacterCoding.html2unicode(stripAllTags(inlineHtml))); // start a new scraper to parse links inside this text @@ -565,10 +559,16 @@ private String recursiveParse(final char[] inlineHtml) { } catch (final IOException e) { } } - for (final Map.Entry entry: scraper.getAnchors().entrySet()) { - mergeAnchors(entry.getKey(), entry.getValue()); + for (final AnchorURL entry: scraper.getAnchors()) { + this.anchors.add(entry); + } + for (ImageEntry ie: scraper.images) { + if (linkurl != null) { + ie.setLinkurl(linkurl); + ie.setAnchortext(new String(inlineHtml)); + } + this.images.add(ie); } - this.images.putAll(scraper.images); String line = cleanLine(CharacterCoding.html2unicode(stripAllTags(scraper.content.getChars()))); scraper.close(); @@ -647,21 +647,21 @@ public String[] getLi() { return this.li.toArray(new String[this.li.size()]); } - public DigestURI[] getFlash() { + public DigestURL[] getFlash() { String ext; - ArrayList f = new ArrayList(); - for (final DigestURI url: this.anchors.keySet()) { - ext = MultiProtocolURI.getFileExtension(url.getFileName()); + ArrayList f = new ArrayList(); + for (final DigestURL url: this.anchors) { + ext = MultiProtocolURL.getFileExtension(url.getFileName()); if (ext == null) continue; if (ext.equals("swf")) f.add(url); } - return f.toArray(new DigestURI[f.size()]); + return f.toArray(new DigestURL[f.size()]); } public boolean containsFlash() { String ext; - for (final MultiProtocolURI url: this.anchors.keySet()) { - ext = MultiProtocolURI.getFileExtension(url.getFileName()); + for (final MultiProtocolURL url: this.anchors) { + ext = MultiProtocolURL.getFileExtension(url.getFileName()); if (ext == null) continue; if (ext.equals("swf")) return true; } @@ -681,48 +681,48 @@ public String getText() { } } - public Map getAnchors() { + public List getAnchors() { // returns a url (String) / name (String) relation return this.anchors; } - public Map getRSS() { + public LinkedHashMap getRSS() { // returns a url (String) / name (String) relation return this.rss; } - public Map getCSS() { + public Map getCSS() { // returns a url (String) / name (String) relation return this.css; } - public Set getFrames() { + public Set getFrames() { // returns a url (String) / name (String) relation return this.frames; } - public Set getIFrames() { + public Set getIFrames() { // returns a url (String) / name (String) relation return this.iframes; } - public Set getScript() { + public Set getScript() { return this.script; } - public DigestURI getCanonical() { + public DigestURL getCanonical() { return this.canonical; } - public DigestURI getPublisherLink() { + public DigestURL getPublisherLink() { return this.publisher; } - public Map getHreflang() { + public Map getHreflang() { return this.hreflang; } - public Map getNavigation() { + public Map getNavigation() { return this.navigation; } @@ -730,11 +730,11 @@ public Map getNavigation() { * get all images * @return a map of */ - public Map getImages() { + public List getImages() { return this.images; } - public Map getEmbeds() { + public Map getEmbeds() { return this.embeds; } @@ -743,9 +743,9 @@ public Map getMetas() { } /** - * @return the {@link MultiProtocolURI} to the favicon that belongs to the document + * @return the {@link MultiProtocolURL} to the favicon that belongs to the document */ - public MultiProtocolURI getFavicon() { + public MultiProtocolURL getFavicon() { return this.favicon; } @@ -1013,36 +1013,18 @@ public static ContentScraper parseResource(final File file, final int maxLinks) if (page == null) throw new IOException("no content in file " + file.toString()); // scrape document to look up charset - final ScraperInputStream htmlFilter = new ScraperInputStream(new ByteArrayInputStream(page),"UTF-8", new DigestURI("http://localhost"),null,false, maxLinks); + final ScraperInputStream htmlFilter = new ScraperInputStream(new ByteArrayInputStream(page),"UTF-8", new DigestURL("http://localhost"),null,false, maxLinks); String charset = htmlParser.patchCharsetEncoding(htmlFilter.detectCharset()); htmlFilter.close(); if (charset == null) charset = Charset.defaultCharset().toString(); // scrape content - final ContentScraper scraper = new ContentScraper(new DigestURI("http://localhost"), maxLinks); + final ContentScraper scraper = new ContentScraper(new DigestURL("http://localhost"), maxLinks); final Writer writer = new TransformerWriter(null, null, scraper, null, false); FileUtils.copy(new ByteArrayInputStream(page), writer, Charset.forName(charset)); writer.close(); return scraper; } - public static void addAllImages(final Map a, final Map b) { - final Iterator> i = b.entrySet().iterator(); - Map.Entry ie; - while (i.hasNext()) { - ie = i.next(); - addImage(a, ie.getValue()); - } - } - - public static void addImage(final Map a, final ImageEntry ie) { - if (a.containsKey(ie.url())) { - // in case of a collision, take that image that has the better image size tags - if ((ie.height() > 0) && (ie.width() > 0)) a.put(ie.url(), ie); - } else { - a.put(ie.url(), ie); - } - } - } diff --git a/source/net/yacy/document/parser/html/ContentTransformer.java b/source/net/yacy/document/parser/html/ContentTransformer.java index 22f61a088d..e4dbf62385 100644 --- a/source/net/yacy/document/parser/html/ContentTransformer.java +++ b/source/net/yacy/document/parser/html/ContentTransformer.java @@ -32,7 +32,7 @@ import java.util.Properties; import java.util.TreeSet; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.kelondro.io.CharBuffer; public class ContentTransformer extends AbstractTransformer implements Transformer { diff --git a/source/net/yacy/document/parser/html/EmbedEntry.java b/source/net/yacy/document/parser/html/EmbedEntry.java index cbaaffb2a0..36134a2738 100644 --- a/source/net/yacy/document/parser/html/EmbedEntry.java +++ b/source/net/yacy/document/parser/html/EmbedEntry.java @@ -20,15 +20,15 @@ package net.yacy.document.parser.html; -import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.cora.document.id.DigestURL; public class EmbedEntry { - private final DigestURI url; + private final DigestURL url; private final int width, height; private final String type, pluginspage; - public EmbedEntry(final DigestURI url, int width, int height, String type, String pluginspage) { + public EmbedEntry(final DigestURL url, int width, int height, String type, String pluginspage) { this.url = url; this.width = width; this.height = height; @@ -36,7 +36,7 @@ public EmbedEntry(final DigestURI url, int width, int height, String type, Strin this.pluginspage = pluginspage; } - public DigestURI getUrl() { + public DigestURL getUrl() { return this.url; } diff --git a/source/net/yacy/document/parser/html/ImageEntry.java b/source/net/yacy/document/parser/html/ImageEntry.java index 37419fffc9..f1d160f055 100644 --- a/source/net/yacy/document/parser/html/ImageEntry.java +++ b/source/net/yacy/document/parser/html/ImageEntry.java @@ -26,26 +26,62 @@ import java.util.Comparator; -import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.cora.document.id.DigestURL; public class ImageEntry implements Comparable, Comparator { - private final DigestURI url; + private final DigestURL imageurl; + private DigestURL linkurl; private final String alt; + private String anchortext; private final int width, height; private final long fileSize; - public ImageEntry(final DigestURI url, final String alt, final int width, final int height, long fileSize) { - assert url != null; - this.url = url; + /** + * an ImageEntry represents the appearance of an image in a document. It considers also that an image can be used as an button for a web link + * and stores the web link also. + * @param imageurl the link to the image + * @param linkurl the link which is called when the image is pressed on a web browser. null if the image was not used as link button + * @param anchortext the text inside the anchor body where the image link appears (including the image tag). null if the image was not used as link button + * @param alt the als text in the alt tag + * @param width the width of the image if known, or -1 if unknown + * @param height the height of the image if known, or -1 if unknown + * @param fileSize the number of bytes that the image uses on file or -1 if unknown + */ + public ImageEntry( + final DigestURL imageurl, + final String alt, + final int width, + final int height, + long fileSize) { + assert imageurl != null; + this.imageurl = imageurl; + this.linkurl = null; + this.anchortext = null; this.alt = alt; this.width = width; this.height = height; this.fileSize = fileSize; } - public DigestURI url() { - return this.url; + public DigestURL url() { + return this.imageurl; + } + + public void setLinkurl(DigestURL linkurl) { + this.linkurl = linkurl; + } + + public DigestURL linkurl() { + return this.linkurl; + } + + public void setAnchortext(String anchortext) { + this.anchortext = anchortext; + } + + public String anchortext() { + return this.anchortext; } public String alt() { @@ -66,7 +102,8 @@ public long fileSize() { @Override public String toString() { - return " 0 ? " alt=\"" + this.alt + "\"" : "") + (this.width >= 0 ? " width=\"" + this.width + "\"" : "") + (this.height >= 0 ? " height=\"" + this.height + "\"" : "") + @@ -80,8 +117,8 @@ public int hashCode() { // unfortunately it can not be ensured that all images get different hashes, but this should appear // only in very rare cases if (this.width < 0 || this.height < 0) - return /*0x7FFF0000 |*/ (this.url.hashCode() & 0xFFFF); - return ((0x7FFF - (((this.width * this.height) >> 9) & 0x7FFF)) << 16) | (this.url.hashCode() & 0xFFFF); + return /*0x7FFF0000 |*/ (this.imageurl.hashCode() & 0xFFFF); + return ((0x7FFF - (((this.width * this.height) >> 9) & 0x7FFF)) << 16) | (this.imageurl.hashCode() & 0xFFFF); } @Override @@ -90,13 +127,13 @@ public int compareTo(final ImageEntry h) { // this method uses the image-size ordering from the hashCode method // assuming that hashCode would return a 'perfect hash' this method would // create a total ordering on images with respect on the image size - assert (this.url != null); - if (this.url.toNormalform(true).equals((h).url.toNormalform(true))) return 0; + assert (this.imageurl != null); + if (this.imageurl.toNormalform(true).equals((h).imageurl.toNormalform(true))) return 0; final int thc = this.hashCode(); final int ohc = (h).hashCode(); if (thc < ohc) return -1; if (thc > ohc) return 1; - return this.url.toString().compareTo((h).url.toString()); + return this.imageurl.toString().compareTo((h).imageurl.toString()); } @Override diff --git a/source/net/yacy/document/parser/html/ScraperInputStream.java b/source/net/yacy/document/parser/html/ScraperInputStream.java index a19b1388c0..ac2a763f87 100644 --- a/source/net/yacy/document/parser/html/ScraperInputStream.java +++ b/source/net/yacy/document/parser/html/ScraperInputStream.java @@ -35,7 +35,7 @@ import java.io.Writer; import java.util.Properties; -import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.cora.document.id.DigestURL; public class ScraperInputStream extends InputStream implements ScraperListener { @@ -58,7 +58,7 @@ public class ScraperInputStream extends InputStream implements ScraperListener { public ScraperInputStream( final InputStream inStream, final String inputStreamCharset, - final DigestURI rooturl, + final DigestURL rooturl, final Transformer transformer, final boolean passbyIfBinarySuspect, final int maxLinks diff --git a/source/net/yacy/document/parser/html/TransformerWriter.java b/source/net/yacy/document/parser/html/TransformerWriter.java index 175ac19349..f26161be88 100644 --- a/source/net/yacy/document/parser/html/TransformerWriter.java +++ b/source/net/yacy/document/parser/html/TransformerWriter.java @@ -44,9 +44,9 @@ import java.util.Enumeration; import java.util.Properties; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.io.CharBuffer; @@ -545,7 +545,7 @@ public static void main(final String[] args) { System.exit(0); final char[] buffer = new char[512]; try { - final ContentScraper scraper = new ContentScraper(new DigestURI("http://localhost:8090"), 1000); + final ContentScraper scraper = new ContentScraper(new DigestURL("http://localhost:8090"), 1000); final Transformer transformer = new ContentTransformer(); final Reader is = new FileReader(args[0]); final FileOutputStream fos = new FileOutputStream(new File(args[0] + ".out")); diff --git a/source/net/yacy/document/parser/htmlParser.java b/source/net/yacy/document/parser/htmlParser.java index 2b5c05f93d..ca1fd935fe 100644 --- a/source/net/yacy/document/parser/htmlParser.java +++ b/source/net/yacy/document/parser/htmlParser.java @@ -32,17 +32,19 @@ import java.nio.charset.Charset; import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.UnsupportedCharsetException; +import java.util.LinkedHashMap; import java.util.regex.Pattern; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.parser.html.CharacterCoding; import net.yacy.document.parser.html.ContentScraper; +import net.yacy.document.parser.html.ImageEntry; import net.yacy.document.parser.html.ScraperInputStream; import net.yacy.document.parser.html.TransformerWriter; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import com.ibm.icu.text.CharsetDetector; @@ -86,7 +88,7 @@ public htmlParser() { @Override public Document[] parse( - final DigestURI location, + final DigestURL location, final String mimeType, final String documentCharset, final InputStream sourceStream) throws Parser.Failure, InterruptedException { @@ -110,7 +112,7 @@ public Document[] parse( * @param scraper * @return */ - private static Document transformScraper(final DigestURI location, final String mimeType, final String charSet, final ContentScraper scraper) { + private static Document transformScraper(final DigestURL location, final String mimeType, final String charSet, final ContentScraper scraper) { final String[] sections = new String[ scraper.getHeadlines(1).length + scraper.getHeadlines(2).length + @@ -124,6 +126,8 @@ private static Document transformScraper(final DigestURI location, final String sections[p++] = headline; } } + LinkedHashMap noDoubleImages = new LinkedHashMap(); + for (ImageEntry ie: scraper.getImages()) noDoubleImages.put(ie.url(), ie); final Document ppd = new Document( location, mimeType, @@ -140,7 +144,7 @@ private static Document transformScraper(final DigestURI location, final String scraper.getText(), scraper.getAnchors(), scraper.getRSS(), - scraper.getImages(), + noDoubleImages, scraper.indexingDenied(), scraper.getDate()); ppd.setFavicon(scraper.getFavicon()); @@ -149,7 +153,7 @@ private static Document transformScraper(final DigestURI location, final String } public static ContentScraper parseToScraper( - final DigestURI location, + final DigestURL location, final String documentCharset, InputStream sourceStream, final int maxLinks) throws Parser.Failure, IOException { @@ -297,9 +301,9 @@ public static String patchCharsetEncoding(String encoding) { public static void main(final String[] args) { // test parsing of a url - DigestURI url; + DigestURL url; try { - url = new DigestURI(args[0]); + url = new DigestURL(args[0]); final byte[] content = url.get(ClientIdentification.yacyInternetCrawlerAgent); final Document[] document = new htmlParser().parse(url, "text/html", null, new ByteArrayInputStream(content)); final String title = document[0].dc_title(); diff --git a/source/net/yacy/document/parser/images/genericImageParser.java b/source/net/yacy/document/parser/images/genericImageParser.java index 39bf0df8d1..7deeb71b7f 100644 --- a/source/net/yacy/document/parser/images/genericImageParser.java +++ b/source/net/yacy/document/parser/images/genericImageParser.java @@ -39,20 +39,21 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; -import java.util.Properties; import java.util.Set; import javax.imageio.ImageIO; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.AnchorURL; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.parser.html.ImageEntry; import net.yacy.document.parser.images.bmpParser.IMAGEMAP; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import com.drew.imaging.jpeg.JpegProcessingException; @@ -94,7 +95,7 @@ public genericImageParser() { @Override public Document[] parse( - final DigestURI location, + final DigestURL location, final String mimeType, final String documentCharset, final InputStream sourceStream) throws Parser.Failure, InterruptedException { @@ -105,7 +106,7 @@ public Document[] parse( String keywords = null; List descriptions = new ArrayList(); String filename = location.getFileName(); - String ext = MultiProtocolURI.getFileExtension(filename); + String ext = MultiProtocolURL.getFileExtension(filename); double gpslat = 0; double gpslon = 0; if (mimeType.equals("image/bmp") || ext.equals("bmp")) { @@ -197,13 +198,13 @@ public Document[] parse( } final HashSet languages = new HashSet(); - final HashMap anchors = new HashMap(); - final HashMap images = new HashMap(); + final List anchors = new ArrayList(); + final LinkedHashMap images = new LinkedHashMap(); // add this image to the map of images final String infoString = ii.info.toString(); images.put(ii.location, new ImageEntry(location, "", ii.width, ii.height, -1)); - if (title == null || title.isEmpty()) title = MultiProtocolURI.unescape(filename); + if (title == null || title.isEmpty()) title = MultiProtocolURL.unescape(filename); return new Document[]{new Document( location, @@ -237,7 +238,7 @@ public Set supportedExtensions() { } public static ImageInfo parseJavaImage( - final DigestURI location, + final DigestURL location, final InputStream sourceStream) throws Parser.Failure { BufferedImage image = null; try { @@ -252,7 +253,7 @@ public static ImageInfo parseJavaImage( } public static ImageInfo parseJavaImage( - final DigestURI location, + final DigestURL location, final BufferedImage image) { final ImageInfo ii = new ImageInfo(location); ii.image = image; @@ -289,12 +290,12 @@ public static ImageInfo parseJavaImage( } public static class ImageInfo { - public DigestURI location; + public DigestURL location; public BufferedImage image; public StringBuilder info; public int height; public int width; - public ImageInfo(final DigestURI location) { + public ImageInfo(final DigestURL location) { this.location = location; this.image = null; this.info = new StringBuilder(); @@ -308,10 +309,10 @@ public ImageInfo(final DigestURI location) { public static void main(final String[] args) { final File image = new File(args[0]); final genericImageParser parser = new genericImageParser(); - DigestURI uri; + DigestURL uri; try { - uri = new DigestURI("http://localhost/" + image.getName()); - final Document[] document = parser.parse(uri, "image/" + MultiProtocolURI.getFileExtension(uri.getFileName()), "UTF-8", new FileInputStream(image)); + uri = new DigestURL("http://localhost/" + image.getName()); + final Document[] document = parser.parse(uri, "image/" + MultiProtocolURL.getFileExtension(uri.getFileName()), "UTF-8", new FileInputStream(image)); System.out.println(document[0].toString()); } catch (final MalformedURLException e) { e.printStackTrace(); diff --git a/source/net/yacy/document/parser/mmParser.java b/source/net/yacy/document/parser/mmParser.java index 773e0a71fc..a0bcc6cebf 100644 --- a/source/net/yacy/document/parser/mmParser.java +++ b/source/net/yacy/document/parser/mmParser.java @@ -34,11 +34,11 @@ import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; import org.xml.sax.Attributes; import org.xml.sax.SAXException; @@ -70,7 +70,7 @@ private static SAXParser getParser() throws SAXException { } @Override - public Document[] parse(final DigestURI location, final String mimeType, + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { diff --git a/source/net/yacy/document/parser/odtParser.java b/source/net/yacy/document/parser/odtParser.java index 19dcab095e..c30f03c62f 100644 --- a/source/net/yacy/document/parser/odtParser.java +++ b/source/net/yacy/document/parser/odtParser.java @@ -42,13 +42,13 @@ import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.parser.xml.ODContentHandler; import net.yacy.document.parser.xml.ODMetaHandler; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.io.CharBuffer; import net.yacy.kelondro.util.FileUtils; @@ -107,7 +107,7 @@ private static SAXParser getParser() throws SAXException { return parser; } - private Document[] parse(final DigestURI location, final String mimeType, @SuppressWarnings("unused") final String charset, final File dest) + private Document[] parse(final DigestURL location, final String mimeType, @SuppressWarnings("unused") final String charset, final File dest) throws Parser.Failure, InterruptedException { CharBuffer writer = null; @@ -214,7 +214,7 @@ private Document[] parse(final DigestURI location, final String mimeType, @Suppr } @Override - public Document[] parse(final DigestURI location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { File dest = null; try { // creating a tempfile diff --git a/source/net/yacy/document/parser/ooxmlParser.java b/source/net/yacy/document/parser/ooxmlParser.java index 0f473956da..d4a52ae249 100644 --- a/source/net/yacy/document/parser/ooxmlParser.java +++ b/source/net/yacy/document/parser/ooxmlParser.java @@ -42,14 +42,14 @@ import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.parser.xml.ODContentHandler; import net.yacy.document.parser.xml.ODMetaHandler; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.io.CharBuffer; import net.yacy.kelondro.util.FileUtils; @@ -91,7 +91,7 @@ private static SAXParser getParser() throws SAXException { return parser; } - private Document[] parse(final DigestURI location, final String mimeType, @SuppressWarnings("unused") final String charset, final File dest) throws Parser.Failure, InterruptedException { + private Document[] parse(final DigestURL location, final String mimeType, @SuppressWarnings("unused") final String charset, final File dest) throws Parser.Failure, InterruptedException { CharBuffer writer = null; try { @@ -201,7 +201,7 @@ private Document[] parse(final DigestURI location, final String mimeType, @Suppr } @Override - public Document[] parse(final DigestURI location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { File dest = null; try { // creating a tempfile diff --git a/source/net/yacy/document/parser/pdfParser.java b/source/net/yacy/document/parser/pdfParser.java index dbf6d19918..b759798ecb 100644 --- a/source/net/yacy/document/parser/pdfParser.java +++ b/source/net/yacy/document/parser/pdfParser.java @@ -55,12 +55,12 @@ import org.apache.pdfbox.pdmodel.font.PDType3Font; import org.apache.pdfbox.util.PDFTextStripper; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.io.CharBuffer; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.MemoryControl; @@ -84,7 +84,7 @@ public pdfParser() { } @Override - public Document[] parse(final DigestURI location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { // check memory for parser if (!MemoryControl.request(200 * 1024 * 1024, false)) @@ -141,7 +141,7 @@ public Document[] parse(final DigestURI location, final String mimeType, final S info = null; if (docTitle == null || docTitle.isEmpty()) { - docTitle = MultiProtocolURI.unescape(location.getFileName()); + docTitle = MultiProtocolURL.unescape(location.getFileName()); } final CharBuffer writer = new CharBuffer(odtParser.MAX_DOCSIZE); byte[] contentBytes = new byte[0]; diff --git a/source/net/yacy/document/parser/pptParser.java b/source/net/yacy/document/parser/pptParser.java index 59c7706271..4f0128b6a9 100644 --- a/source/net/yacy/document/parser/pptParser.java +++ b/source/net/yacy/document/parser/pptParser.java @@ -31,11 +31,11 @@ import java.io.InputStream; import java.util.Date; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; import org.apache.poi.hslf.extractor.PowerPointExtractor; @@ -60,7 +60,7 @@ public pptParser(){ * all extracted information about the parsed document */ @Override - public Document[] parse(final DigestURI location, final String mimeType, + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { try { diff --git a/source/net/yacy/document/parser/psParser.java b/source/net/yacy/document/parser/psParser.java index f112927e20..707b7a3ede 100644 --- a/source/net/yacy/document/parser/psParser.java +++ b/source/net/yacy/document/parser/psParser.java @@ -36,10 +36,10 @@ import java.io.InputStreamReader; import java.util.Date; +import net.yacy.cora.document.id.DigestURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; @@ -84,7 +84,7 @@ private boolean testForPs2Ascii() { } - private Document[] parse(final DigestURI location, final String mimeType, @SuppressWarnings("unused") final String charset, final File sourceFile) throws Parser.Failure, InterruptedException { + private Document[] parse(final DigestURL location, final String mimeType, @SuppressWarnings("unused") final String charset, final File sourceFile) throws Parser.Failure, InterruptedException { File outputFile = null; try { @@ -256,7 +256,7 @@ private void parseUsingPS2ascii(final File inputFile, final File outputFile) thr } @Override - public Document[] parse(final DigestURI location, final String mimeType, + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { diff --git a/source/net/yacy/document/parser/rdfParser.java b/source/net/yacy/document/parser/rdfParser.java index d14dacc37b..5079f0475f 100644 --- a/source/net/yacy/document/parser/rdfParser.java +++ b/source/net/yacy/document/parser/rdfParser.java @@ -30,10 +30,10 @@ import java.util.Date; import java.util.List; +import net.yacy.cora.document.id.DigestURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; public class rdfParser extends AbstractParser implements Parser { @@ -45,7 +45,7 @@ public rdfParser() { } @Override - public Document[] parse(final DigestURI url, final String mimeType, + public Document[] parse(final DigestURL url, final String mimeType, final String charset, final InputStream source) throws Failure, InterruptedException { diff --git a/source/net/yacy/document/parser/rdfa/impl/RDFaParser.java b/source/net/yacy/document/parser/rdfa/impl/RDFaParser.java index 0820cd84c2..cf8accb706 100644 --- a/source/net/yacy/document/parser/rdfa/impl/RDFaParser.java +++ b/source/net/yacy/document/parser/rdfa/impl/RDFaParser.java @@ -17,13 +17,13 @@ import java.util.HashSet; import java.util.Set; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.parser.htmlParser; import net.yacy.document.parser.rdfa.IRDFaTriple; -import net.yacy.kelondro.data.meta.DigestURI; /** * @author fgandon @@ -46,7 +46,7 @@ public RDFaParser() { } @Override - public Document[] parse(DigestURI url, String mimeType, + public Document[] parse(DigestURL url, String mimeType, String charset, InputStream source) throws Failure, InterruptedException { @@ -67,7 +67,7 @@ public Document[] parse(DigestURI url, String mimeType, return htmlDocs; } - private static Document parseRDFa(DigestURI url, String mimeType, + private static Document parseRDFa(DigestURL url, String mimeType, String charset, InputStream source) { RDFaTripleImpl triple; IRDFaTriple[] allTriples = null; @@ -95,7 +95,7 @@ private static Document parseRDFa(DigestURI url, String mimeType, return doc; } - private Document[] parseHtml(DigestURI url, String mimeType, + private Document[] parseHtml(DigestURL url, String mimeType, String charset, InputStream source) throws Failure, InterruptedException { @@ -110,7 +110,7 @@ private Document[] parseHtml(DigestURI url, String mimeType, return htmlDocs; } - private static Document convertAllTriplesToDocument(DigestURI url, + private static Document convertAllTriplesToDocument(DigestURL url, String mimeType, String charset, IRDFaTriple[] allTriples) { //Set languages = new HashSet(2); @@ -178,7 +178,7 @@ public static void main(String[] args) { if (aReader != null) { RDFaParser aParser = new RDFaParser(); try { - aParser.parse(new DigestURI(args[0]),"","",aURL.openStream()); + aParser.parse(new DigestURL(args[0]),"","",aURL.openStream()); } catch (final FileNotFoundException e) { e.printStackTrace(); } catch (final IOException e) { diff --git a/source/net/yacy/document/parser/rssParser.java b/source/net/yacy/document/parser/rssParser.java index af8cb186ed..2318839043 100644 --- a/source/net/yacy/document/parser/rssParser.java +++ b/source/net/yacy/document/parser/rssParser.java @@ -29,22 +29,21 @@ import java.io.InputStream; import java.net.MalformedURLException; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; -import java.util.Properties; import java.util.Set; -import net.yacy.cora.document.Hit; -import net.yacy.cora.document.RSSFeed; -import net.yacy.cora.document.RSSReader; +import net.yacy.cora.document.feed.Hit; +import net.yacy.cora.document.feed.RSSFeed; +import net.yacy.cora.document.feed.RSSReader; +import net.yacy.cora.document.id.AnchorURL; +import net.yacy.cora.document.id.DigestURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.TextParser; import net.yacy.document.parser.html.ImageEntry; -import net.yacy.kelondro.data.meta.DigestURI; public class rssParser extends AbstractParser implements Parser { @@ -59,7 +58,7 @@ public rssParser() { } @Override - public Document[] parse(final DigestURI url, final String mimeType, + public Document[] parse(final DigestURL url, final String mimeType, final String charset, final InputStream source) throws Failure, InterruptedException { RSSReader rssReader; @@ -72,18 +71,17 @@ public Document[] parse(final DigestURI url, final String mimeType, final RSSFeed feed = rssReader.getFeed(); //RSSMessage channel = feed.getChannel(); final List docs = new ArrayList(); - DigestURI uri; + AnchorURL uri; Set languages; - Map anchors; + List anchors; Document doc; for (final Hit item: feed) try { - uri = new DigestURI(item.getLink()); + uri = new AnchorURL(item.getLink()); languages = new HashSet(); languages.add(item.getLanguage()); - anchors = new HashMap(); - Properties p = new Properties(); - p.put("name", item.getTitle()); - anchors.put(uri, p); + anchors = new ArrayList(); + uri.getProperties().put("name", item.getTitle()); + anchors.add(uri); doc = new Document( uri, TextParser.mimeOf(url), @@ -101,7 +99,7 @@ public Document[] parse(final DigestURI url, final String mimeType, null, anchors, null, - new HashMap(), + new LinkedHashMap(), false, item.getPubDate()); docs.add(doc); diff --git a/source/net/yacy/document/parser/rtfParser.java b/source/net/yacy/document/parser/rtfParser.java index 1a1495aa76..1ac87a76a1 100644 --- a/source/net/yacy/document/parser/rtfParser.java +++ b/source/net/yacy/document/parser/rtfParser.java @@ -33,10 +33,10 @@ import javax.swing.text.DefaultStyledDocument; import javax.swing.text.rtf.RTFEditorKit; +import net.yacy.cora.document.id.DigestURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; public class rtfParser extends AbstractParser implements Parser { @@ -52,7 +52,7 @@ public rtfParser() { } @Override - public Document[] parse(final DigestURI location, final String mimeType, + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { diff --git a/source/net/yacy/document/parser/sevenzipParser.java b/source/net/yacy/document/parser/sevenzipParser.java index b7c7464d0a..b58d7e6a71 100644 --- a/source/net/yacy/document/parser/sevenzipParser.java +++ b/source/net/yacy/document/parser/sevenzipParser.java @@ -34,12 +34,12 @@ import java.io.OutputStream; import java.util.Date; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.TextParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import SevenZip.ArchiveExtractCallback; import SevenZip.IInStream; @@ -55,7 +55,7 @@ public sevenzipParser() { this.SUPPORTED_MIME_TYPES.add("application/x-7z-compressed"); } - public Document parse(final DigestURI location, final String mimeType, final String charset, final IInStream source) throws Parser.Failure, InterruptedException { + public Document parse(final DigestURL location, final String mimeType, final String charset, final IInStream source) throws Parser.Failure, InterruptedException { final Document doc = new Document( location, mimeType, @@ -100,12 +100,12 @@ public Document parse(final DigestURI location, final String mimeType, final Str } } - public Document parse(final DigestURI location, final String mimeType, final String charset, final byte[] source) throws Parser.Failure, InterruptedException { + public Document parse(final DigestURL location, final String mimeType, final String charset, final byte[] source) throws Parser.Failure, InterruptedException { return parse(location, mimeType, charset, new ByteArrayIInStream(source)); } @Override - public Document[] parse(final DigestURI location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { try { final ByteArrayOutputStream cfos = new ByteArrayOutputStream(); FileUtils.copy(source, cfos); @@ -169,7 +169,7 @@ public void SetOperationResult(final int arg0) throws IOException { Document[] theDocs; // workaround for relative links in file, normally '#' shall be used behind the location, see // below for reversion of the effects - final DigestURI url = DigestURI.newURL(this.doc.dc_source(), this.prefix + "/" + super.filePath); + final DigestURL url = DigestURL.newURL(this.doc.dc_source(), this.prefix + "/" + super.filePath); final String mime = TextParser.mimeOf(super.filePath.substring(super.filePath.lastIndexOf('.') + 1)); theDocs = TextParser.parseSource(url, mime, null, this.cfos.toByteArray()); diff --git a/source/net/yacy/document/parser/sidAudioParser.java b/source/net/yacy/document/parser/sidAudioParser.java index aa3cf643d4..799ef60a2f 100644 --- a/source/net/yacy/document/parser/sidAudioParser.java +++ b/source/net/yacy/document/parser/sidAudioParser.java @@ -31,10 +31,10 @@ import java.util.HashMap; import java.util.Map; +import net.yacy.cora.document.id.DigestURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; // this is a new implementation of this parser idiom using multiple documents as result set @@ -57,7 +57,7 @@ public sidAudioParser() { } @Override - public Document[] parse(final DigestURI location, final String mimeType, + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { try { diff --git a/source/net/yacy/document/parser/sitemapParser.java b/source/net/yacy/document/parser/sitemapParser.java index ce42595faf..17a472f6da 100644 --- a/source/net/yacy/document/parser/sitemapParser.java +++ b/source/net/yacy/document/parser/sitemapParser.java @@ -31,7 +31,7 @@ import java.text.ParseException; import java.util.ArrayList; import java.util.Date; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -40,6 +40,7 @@ import javax.xml.parsers.DocumentBuilderFactory; import net.yacy.cora.date.ISO8601Formatter; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.ResponseHeader; @@ -50,7 +51,6 @@ import net.yacy.document.Parser; import net.yacy.document.TextParser; import net.yacy.document.parser.html.ImageEntry; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.io.ByteCountInputStream; import org.w3c.dom.CharacterData; @@ -68,17 +68,17 @@ public sitemapParser() { } @Override - public Document[] parse(final DigestURI url, final String mimeType, + public Document[] parse(final DigestURL url, final String mimeType, final String charset, final InputStream source) throws Failure, InterruptedException { final List docs = new ArrayList(); SitemapReader sitemap = new SitemapReader(source, ClientIdentification.yacyInternetCrawlerAgent); sitemap.start(); - DigestURI uri; + DigestURL uri; Document doc; URLEntry item; while ((item = sitemap.take()) != POISON_URLEntry) try { - uri = new DigestURI(item.loc); + uri = new DigestURL(item.loc); doc = new Document( uri, TextParser.mimeOf(url), @@ -95,7 +95,7 @@ public Document[] parse(final DigestURI url, final String mimeType, null, null, null, - new HashMap(), + new LinkedHashMap(), false, new Date()); docs.add(doc); @@ -108,7 +108,7 @@ public Document[] parse(final DigestURI url, final String mimeType, return da; } - public static SitemapReader parse(final DigestURI sitemapURL, final ClientIdentification.Agent agent) throws IOException { + public static SitemapReader parse(final DigestURL sitemapURL, final ClientIdentification.Agent agent) throws IOException { // download document ConcurrentLog.info("SitemapReader", "loading sitemap from " + sitemapURL.toNormalform(true)); final RequestHeader requestHeader = new RequestHeader(); @@ -160,7 +160,7 @@ public void run() { String url = new SitemapEntry((Element) sitemapNodes.item(i)).url(); if (url != null && url.length() > 0) { try { - final SitemapReader r = parse(new DigestURI(url), agent); + final SitemapReader r = parse(new DigestURL(url), agent); r.start(); URLEntry item; while ((item = r.take()) != POISON_URLEntry) { diff --git a/source/net/yacy/document/parser/swfParser.java b/source/net/yacy/document/parser/swfParser.java index 311420a8a2..6be3f8edb6 100644 --- a/source/net/yacy/document/parser/swfParser.java +++ b/source/net/yacy/document/parser/swfParser.java @@ -31,15 +31,13 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Properties; +import net.yacy.cora.document.id.AnchorURL; +import net.yacy.cora.document.id.DigestURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; import pt.tumba.parser.swf.SWF2HTML; public class swfParser extends AbstractParser implements Parser { @@ -58,7 +56,7 @@ public swfParser() { * all extracted information about the parsed document */ @Override - public Document[] parse(final DigestURI location, final String mimeType, + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { @@ -81,7 +79,7 @@ public Document[] parse(final DigestURI location, final String mimeType, final String[] sections = null; final List abstrct = new ArrayList(); //TreeSet images = null; - final Map anchors = new HashMap(); + final List anchors = new ArrayList(); int urls = 0; int urlStart = -1; int urlEnd = 0; @@ -98,9 +96,9 @@ public Document[] parse(final DigestURI location, final String mimeType, urlEnd = contents.indexOf(linebreak,urlStart); url = contents.substring(urlStart,urlEnd); urlnr = Integer.toString(++urls).toString(); - final Properties p = new Properties(); - p.put("name", urlnr); - anchors.put(new DigestURI(url), p); + AnchorURL u = new AnchorURL(url); + u.getProperties().put("name", urlnr); + anchors.add(u); contents = contents.substring(0,urlStart)+contents.substring(urlEnd); } diff --git a/source/net/yacy/document/parser/tarParser.java b/source/net/yacy/document/parser/tarParser.java index df8f32f66d..f70715218b 100644 --- a/source/net/yacy/document/parser/tarParser.java +++ b/source/net/yacy/document/parser/tarParser.java @@ -33,13 +33,13 @@ import java.util.List; import java.util.zip.GZIPInputStream; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.TextParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import org.apache.tools.tar.TarEntry; @@ -61,11 +61,11 @@ public tarParser() { } @Override - public Document[] parse(final DigestURI url, final String mimeType, final String charset, InputStream source) throws Parser.Failure, InterruptedException { + public Document[] parse(final DigestURL url, final String mimeType, final String charset, InputStream source) throws Parser.Failure, InterruptedException { final List docacc = new ArrayList(); Document[] subDocs = null; - final String ext = MultiProtocolURI.getFileExtension(url.getFileName()).toLowerCase(); + final String ext = MultiProtocolURL.getFileExtension(url.getFileName()).toLowerCase(); if (ext.equals("gz") || ext.equals("tgz")) { try { source = new GZIPInputStream(source); @@ -90,7 +90,7 @@ public Document[] parse(final DigestURI url, final String mimeType, final String try { tmp = FileUtils.createTempFile(this.getClass(), name); FileUtils.copy(tis, tmp, entry.getSize()); - subDocs = TextParser.parseSource(DigestURI.newURL(url, "#" + name), mime, null, tmp); + subDocs = TextParser.parseSource(DigestURL.newURL(url, "#" + name), mime, null, tmp); if (subDocs == null) continue; for (final Document d: subDocs) docacc.add(d); } catch (final Parser.Failure e) { diff --git a/source/net/yacy/document/parser/torrentParser.java b/source/net/yacy/document/parser/torrentParser.java index 4dcf33d4d8..5c3ff5d1f6 100644 --- a/source/net/yacy/document/parser/torrentParser.java +++ b/source/net/yacy/document/parser/torrentParser.java @@ -32,14 +32,14 @@ import java.util.List; import java.util.Map; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.document.AbstractParser; import net.yacy.document.Condenser; import net.yacy.document.Document; import net.yacy.document.LibraryProvider; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.util.BDecoder; import net.yacy.kelondro.util.BDecoder.BObject; @@ -56,7 +56,7 @@ public torrentParser() { } @Override - public Document[] parse(DigestURI location, String mimeType, String charset, InputStream source) + public Document[] parse(DigestURL location, String mimeType, String charset, InputStream source) throws Parser.Failure, InterruptedException { byte[] b = null; try { @@ -93,7 +93,7 @@ public Document[] parse(DigestURI location, String mimeType, String charset, Inp final BObject nameo = info.get("name"); if (nameo != null) title = UTF8.String(nameo.getString()); } - if (title == null || title.isEmpty()) title = MultiProtocolURI.unescape(location.getFileName()); + if (title == null || title.isEmpty()) title = MultiProtocolURL.unescape(location.getFileName()); return new Document[]{new Document( location, mimeType, @@ -119,7 +119,7 @@ public static void main(String[] args) { try { byte[] b = FileUtils.read(new File(args[0])); torrentParser parser = new torrentParser(); - Document[] d = parser.parse(new DigestURI("http://localhost/test.torrent"), null, "UTF-8", new ByteArrayInputStream(b)); + Document[] d = parser.parse(new DigestURL("http://localhost/test.torrent"), null, "UTF-8", new ByteArrayInputStream(b)); Condenser c = new Condenser(d[0], true, true, LibraryProvider.dymLib, LibraryProvider.synonyms, false); Map w = c.words(); for (Map.Entry e: w.entrySet()) System.out.println("Word: " + e.getKey() + " - " + e.getValue().posInText); diff --git a/source/net/yacy/document/parser/vcfParser.java b/source/net/yacy/document/parser/vcfParser.java index 25726314ce..99b46f2812 100644 --- a/source/net/yacy/document/parser/vcfParser.java +++ b/source/net/yacy/document/parser/vcfParser.java @@ -38,14 +38,14 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; -import java.util.Properties; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.AnchorURL; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.order.Base64Order; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; /** * Vcard specification: http://www.imc.org/pdi/vcard-21.txt @@ -65,14 +65,14 @@ public vcfParser() { } @Override - public Document[] parse(final DigestURI url, final String mimeType, final String charset, final InputStream source) + public Document[] parse(final DigestURL url, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { try { final StringBuilder parsedTitle = new StringBuilder(); final StringBuilder parsedDataText = new StringBuilder(); final HashMap parsedData = new HashMap(); - final HashMap anchors = new HashMap(); + final List anchors = new ArrayList(); final LinkedList parsedNames = new LinkedList(); boolean useLastLine = false; @@ -179,10 +179,9 @@ public Document[] parse(final DigestURI url, final String mimeType, final String parsedData.clear(); } else if (key.toUpperCase().startsWith("URL")) { try { - final DigestURI newURL = new DigestURI(value); - final Properties p = new Properties(); - p.put("name", newURL.toString()); - anchors.put(newURL, p); + final AnchorURL newURL = new AnchorURL(value); + newURL.getProperties().put("name", newURL.toString()); + anchors.add(newURL); //parsedData.put(key,value); } catch (final MalformedURLException ex) {/* ignore this */} } else if ( diff --git a/source/net/yacy/document/parser/vsdParser.java b/source/net/yacy/document/parser/vsdParser.java index 6ea7b38851..cd392d28fa 100644 --- a/source/net/yacy/document/parser/vsdParser.java +++ b/source/net/yacy/document/parser/vsdParser.java @@ -32,11 +32,11 @@ import java.util.Date; import java.util.List; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; import org.apache.poi.hdgf.extractor.VisioTextExtractor; import org.apache.poi.hpsf.SummaryInformation; @@ -66,7 +66,7 @@ public vsdParser() { * all extracted information about the parsed document */ @Override - public Document[] parse(final DigestURI location, final String mimeType, final String charset, final InputStream source) + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { Document theDoc = null; diff --git a/source/net/yacy/document/parser/xlsParser.java b/source/net/yacy/document/parser/xlsParser.java index 850f4287a9..ccd3d7d7e0 100644 --- a/source/net/yacy/document/parser/xlsParser.java +++ b/source/net/yacy/document/parser/xlsParser.java @@ -30,11 +30,11 @@ import java.io.InputStream; import java.util.Date; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; -import net.yacy.kelondro.data.meta.DigestURI; import org.apache.poi.hssf.eventusermodel.HSSFEventFactory; import org.apache.poi.hssf.eventusermodel.HSSFListener; @@ -66,7 +66,7 @@ public xlsParser(){ * all extracted information about the parsed document */ @Override - public Document[] parse(final DigestURI location, final String mimeType, + public Document[] parse(final DigestURL location, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { return new XLSHSSFListener().parse(location, mimeType, charset, source); @@ -86,7 +86,7 @@ public XLSHSSFListener() { * parses the source documents and returns a Document containing * all extracted information about the parsed document */ - public Document[] parse(final DigestURI location, final String mimeType, + public Document[] parse(final DigestURL location, final String mimeType, @SuppressWarnings("unused") final String charset, final InputStream source) throws Parser.Failure, InterruptedException { try { diff --git a/source/net/yacy/document/parser/zipParser.java b/source/net/yacy/document/parser/zipParser.java index 48695ae3a8..aaaeb15279 100644 --- a/source/net/yacy/document/parser/zipParser.java +++ b/source/net/yacy/document/parser/zipParser.java @@ -32,11 +32,11 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import net.yacy.cora.document.id.DigestURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.TextParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.MemoryControl; @@ -60,7 +60,7 @@ public zipParser() { } @Override - public Document[] parse(final DigestURI url, final String mimeType, + public Document[] parse(final DigestURL url, final String mimeType, final String charset, final InputStream source) throws Parser.Failure, InterruptedException { // check memory for parser @@ -86,7 +86,7 @@ public Document[] parse(final DigestURI url, final String mimeType, try { tmp = FileUtils.createTempFile(this.getClass(), name); FileUtils.copy(zis, tmp, entry.getSize()); - final DigestURI virtualURL = DigestURI.newURL(url, "#" + name); + final DigestURL virtualURL = DigestURL.newURL(url, "#" + name); //this.log.logInfo("ZIP file parser: " + virtualURL.toNormalform(false, false)); docs = TextParser.parseSource(virtualURL, mime, null, tmp); if (docs == null) continue; diff --git a/source/net/yacy/interaction/AugmentHtmlStream.java b/source/net/yacy/interaction/AugmentHtmlStream.java index 4a79457899..eff2cd833e 100644 --- a/source/net/yacy/interaction/AugmentHtmlStream.java +++ b/source/net/yacy/interaction/AugmentHtmlStream.java @@ -8,13 +8,13 @@ import java.io.StringReader; import java.net.URLEncoder; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import net.yacy.server.http.ServerSideIncludes; @@ -83,7 +83,7 @@ private static String processAddDoctype(String data) { } - public static StringBuilder process(StringBuilder data, DigestURI url, RequestHeader requestHeader) { + public static StringBuilder process(StringBuilder data, DigestURL url, RequestHeader requestHeader) { String action = requestHeader.get("YACYACTION"); requestHeader.remove("YACYACTION"); diff --git a/source/net/yacy/interaction/Interaction.java b/source/net/yacy/interaction/Interaction.java index ae5428b6d9..1bf14f1b63 100644 --- a/source/net/yacy/interaction/Interaction.java +++ b/source/net/yacy/interaction/Interaction.java @@ -8,7 +8,8 @@ import java.util.Map; import java.util.Set; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; @@ -17,7 +18,6 @@ import net.yacy.cora.util.SpaceExceededException; import net.yacy.data.UserDB; import net.yacy.kelondro.blob.Tables.Row; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.peers.Seed; import net.yacy.search.Switchboard; @@ -86,7 +86,7 @@ public static String GetDomain (String url) { String domain = url; try { - DigestURI uri = new DigestURI (url); + DigestURL uri = new DigestURL (url); domain = uri.getHost(); } catch (final MalformedURLException e) { @@ -105,9 +105,9 @@ public static String GetURLHash (String url) { String result = ""; - DigestURI uri; + DigestURL uri; try { - uri = new DigestURI (url); + uri = new DigestURL (url); result = UTF8.String(uri.hash()); diff --git a/source/net/yacy/kelondro/blob/ArrayStack.java b/source/net/yacy/kelondro/blob/ArrayStack.java index 6630d1b49d..7f91a8a3e0 100644 --- a/source/net/yacy/kelondro/blob/ArrayStack.java +++ b/source/net/yacy/kelondro/blob/ArrayStack.java @@ -50,8 +50,8 @@ import java.util.concurrent.TimeUnit; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.order.NaturalOrder; diff --git a/source/net/yacy/kelondro/blob/BEncodedHeap.java b/source/net/yacy/kelondro/blob/BEncodedHeap.java index 455c758f1b..58b4867e3e 100644 --- a/source/net/yacy/kelondro/blob/BEncodedHeap.java +++ b/source/net/yacy/kelondro/blob/BEncodedHeap.java @@ -36,8 +36,8 @@ import java.util.TreeSet; import java.util.regex.Pattern; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.order.CloneableIterator; diff --git a/source/net/yacy/kelondro/blob/BEncodedHeapBag.java b/source/net/yacy/kelondro/blob/BEncodedHeapBag.java index 5a9029737c..3b7ae63c9f 100644 --- a/source/net/yacy/kelondro/blob/BEncodedHeapBag.java +++ b/source/net/yacy/kelondro/blob/BEncodedHeapBag.java @@ -35,7 +35,7 @@ import java.util.Map; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.order.CloneableIterator; diff --git a/source/net/yacy/kelondro/blob/BEncodedHeapShard.java b/source/net/yacy/kelondro/blob/BEncodedHeapShard.java index 8f37235875..180c468774 100644 --- a/source/net/yacy/kelondro/blob/BEncodedHeapShard.java +++ b/source/net/yacy/kelondro/blob/BEncodedHeapShard.java @@ -31,8 +31,8 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.order.CloneableIterator; diff --git a/source/net/yacy/kelondro/blob/Heap.java b/source/net/yacy/kelondro/blob/Heap.java index 019c77fb6d..db4b4bdb66 100644 --- a/source/net/yacy/kelondro/blob/Heap.java +++ b/source/net/yacy/kelondro/blob/Heap.java @@ -34,7 +34,7 @@ import java.util.SortedMap; import java.util.TreeMap; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.order.NaturalOrder; diff --git a/source/net/yacy/kelondro/blob/HeapModifier.java b/source/net/yacy/kelondro/blob/HeapModifier.java index 41db04c2d3..1e0d14a208 100644 --- a/source/net/yacy/kelondro/blob/HeapModifier.java +++ b/source/net/yacy/kelondro/blob/HeapModifier.java @@ -28,7 +28,7 @@ import java.io.IOException; import java.util.SortedMap; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; diff --git a/source/net/yacy/kelondro/blob/HeapReader.java b/source/net/yacy/kelondro/blob/HeapReader.java index 03f5251271..ceacf20781 100644 --- a/source/net/yacy/kelondro/blob/HeapReader.java +++ b/source/net/yacy/kelondro/blob/HeapReader.java @@ -36,8 +36,8 @@ import java.util.Map.Entry; import java.util.concurrent.ExecutionException; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.order.Digest; diff --git a/source/net/yacy/kelondro/blob/HeapWriter.java b/source/net/yacy/kelondro/blob/HeapWriter.java index 7aa3c970cb..6ef001733f 100644 --- a/source/net/yacy/kelondro/blob/HeapWriter.java +++ b/source/net/yacy/kelondro/blob/HeapWriter.java @@ -30,7 +30,7 @@ import java.io.FileOutputStream; import java.io.IOException; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.storage.HandleMap; import net.yacy.cora.util.ConcurrentLog; diff --git a/source/net/yacy/kelondro/blob/MapColumnIndex.java b/source/net/yacy/kelondro/blob/MapColumnIndex.java index e3822cd7a9..f99d1ce05b 100644 --- a/source/net/yacy/kelondro/blob/MapColumnIndex.java +++ b/source/net/yacy/kelondro/blob/MapColumnIndex.java @@ -33,7 +33,7 @@ import java.util.Map; import java.util.TreeMap; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.order.NaturalOrder; /** diff --git a/source/net/yacy/kelondro/blob/MapDataMining.java b/source/net/yacy/kelondro/blob/MapDataMining.java index 9c6fc7d6f5..f82ac01ef3 100644 --- a/source/net/yacy/kelondro/blob/MapDataMining.java +++ b/source/net/yacy/kelondro/blob/MapDataMining.java @@ -38,7 +38,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.order.CloneableIterator; diff --git a/source/net/yacy/kelondro/blob/MapHeap.java b/source/net/yacy/kelondro/blob/MapHeap.java index a69674eac6..f0780e3a3b 100644 --- a/source/net/yacy/kelondro/blob/MapHeap.java +++ b/source/net/yacy/kelondro/blob/MapHeap.java @@ -44,7 +44,7 @@ import java.util.concurrent.ConcurrentHashMap; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.order.NaturalOrder; diff --git a/source/net/yacy/kelondro/blob/Tables.java b/source/net/yacy/kelondro/blob/Tables.java index f8ee3d5965..17c663162b 100644 --- a/source/net/yacy/kelondro/blob/Tables.java +++ b/source/net/yacy/kelondro/blob/Tables.java @@ -44,8 +44,8 @@ import java.util.regex.Pattern; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.util.ByteArray; import net.yacy.cora.util.ByteBuffer; import net.yacy.cora.util.ConcurrentLog; diff --git a/source/net/yacy/kelondro/blob/TablesColumnIndex.java b/source/net/yacy/kelondro/blob/TablesColumnIndex.java index 04f0ffd806..536e4e9f4f 100644 --- a/source/net/yacy/kelondro/blob/TablesColumnIndex.java +++ b/source/net/yacy/kelondro/blob/TablesColumnIndex.java @@ -28,7 +28,7 @@ import java.util.Map; import java.util.Set; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.NaturalOrder; diff --git a/source/net/yacy/kelondro/data/citation/CitationReference.java b/source/net/yacy/kelondro/data/citation/CitationReference.java index 0c2186d476..a72ed2fe87 100644 --- a/source/net/yacy/kelondro/data/citation/CitationReference.java +++ b/source/net/yacy/kelondro/data/citation/CitationReference.java @@ -24,7 +24,7 @@ import java.util.Collection; import net.yacy.cora.date.MicroDate; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.order.Base64Order; import net.yacy.cora.util.ByteArray; import net.yacy.kelondro.data.word.Word; diff --git a/source/net/yacy/kelondro/data/meta/URIMetadataNode.java b/source/net/yacy/kelondro/data/meta/URIMetadataNode.java index b893aaff93..3e3c73b42f 100644 --- a/source/net/yacy/kelondro/data/meta/URIMetadataNode.java +++ b/source/net/yacy/kelondro/data/meta/URIMetadataNode.java @@ -33,11 +33,12 @@ import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.date.MicroDate; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; import net.yacy.cora.document.analysis.Classification; import net.yacy.cora.document.analysis.Classification.ContentDomain; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.solr.SolrType; import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.order.Base64Order; @@ -61,7 +62,7 @@ public class URIMetadataNode { private byte[] hash = null; private String urlRaw = null, keywords = null; - private DigestURI url = null; + private DigestURL url = null; private Bitfield flags = null; private int imagec = -1, audioc = -1, videoc = -1, appc = -1; private double lat = Double.NaN, lon = Double.NaN; @@ -79,7 +80,7 @@ public URIMetadataNode(final SolrDocument doc) { this.hash = ASCII.getBytes(getString(CollectionSchema.id)); this.urlRaw = getString(CollectionSchema.sku); try { - this.url = new DigestURI(this.urlRaw, this.hash); + this.url = new DigestURL(this.urlRaw, this.hash); } catch (final MalformedURLException e) { ConcurrentLog.logException(e); this.url = null; @@ -110,7 +111,7 @@ public Date moddate() { return getDate(CollectionSchema.last_modified); } - public DigestURI url() { + public DigestURL url() { return this.url; } @@ -202,7 +203,7 @@ public Bitfield flags() { if (flags == null) { this.flags = new Bitfield(); if (dc_subject() != null && dc_subject().indexOf("indexof") >= 0) this.flags.set(Condenser.flag_cat_indexof, true); - ContentDomain cd = Classification.getContentDomain(MultiProtocolURI.getFileExtension(this.url().getFileName())); + ContentDomain cd = Classification.getContentDomain(MultiProtocolURL.getFileExtension(this.url().getFileName())); if (lon() != 0.0d || lat() != 0.0d) this.flags.set(Condenser.flag_cat_haslocation, true); if (cd == ContentDomain.IMAGE || limage() > 0) this.flags.set(Condenser.flag_cat_hasimage, true); if (cd == ContentDomain.AUDIO || laudio() > 0) this.flags.set(Condenser.flag_cat_hasaudio, true); diff --git a/source/net/yacy/kelondro/data/meta/URIMetadataRow.java b/source/net/yacy/kelondro/data/meta/URIMetadataRow.java index b1797de388..2bf0716618 100644 --- a/source/net/yacy/kelondro/data/meta/URIMetadataRow.java +++ b/source/net/yacy/kelondro/data/meta/URIMetadataRow.java @@ -34,8 +34,9 @@ import java.util.regex.Pattern; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.Digest; @@ -118,10 +119,10 @@ private URIMetadataRow(final Properties prop) throws kelondroException { // generates an plasmaLURLEntry using the properties from the argument // the property names must correspond to the one from toString //System.out.println("DEBUG-ENTRY: prop=" + prop.toString()); - DigestURI url; + DigestURL url; String urls = crypt.simpleDecode(prop.getProperty("url", "")); try { - url = new DigestURI(urls); + url = new DigestURL(urls); } catch (final MalformedURLException e) { throw new kelondroException("bad url: " + urls); } @@ -210,7 +211,7 @@ private Date decodeDate(final int col) { } private static byte[] encodeComp( - final DigestURI url, + final DigestURL url, final String dc_title, final String dc_creator, final String dc_subject, @@ -252,7 +253,7 @@ public boolean matches(final Pattern matcher) { return this.metadata().matches(matcher); } - public DigestURI url() { + public DigestURL url() { return this.metadata().url(); } @@ -465,7 +466,7 @@ public String toString() { } private class Components { - private DigestURI url; + private DigestURL url; private String urlRaw; private byte[] urlHash; private final String dc_title, dc_creator, dc_subject, dc_publisher; @@ -493,10 +494,10 @@ public boolean matches(final Pattern matcher) { if (this.url != null) return matcher.matcher(this.url.toNormalform(true).toLowerCase()).matches(); return false; } - public DigestURI url() { + public DigestURL url() { if (this.url == null) { try { - this.url = new DigestURI(this.urlRaw, this.urlHash); + this.url = new DigestURL(this.urlRaw, this.urlHash); } catch (final MalformedURLException e) { this.url = null; } diff --git a/source/net/yacy/kelondro/data/navigation/NavigationReferenceRow.java b/source/net/yacy/kelondro/data/navigation/NavigationReferenceRow.java index 18375ddbc6..40bf60454f 100644 --- a/source/net/yacy/kelondro/data/navigation/NavigationReferenceRow.java +++ b/source/net/yacy/kelondro/data/navigation/NavigationReferenceRow.java @@ -28,7 +28,7 @@ import java.util.Collection; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.order.Base64Order; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.index.Column; diff --git a/source/net/yacy/kelondro/data/navigation/NavigationReferenceVars.java b/source/net/yacy/kelondro/data/navigation/NavigationReferenceVars.java index e1771296cd..a0bc1db869 100644 --- a/source/net/yacy/kelondro/data/navigation/NavigationReferenceVars.java +++ b/source/net/yacy/kelondro/data/navigation/NavigationReferenceVars.java @@ -29,7 +29,7 @@ import java.io.Serializable; import java.util.Collection; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.kelondro.index.Row.Entry; import net.yacy.kelondro.rwi.AbstractReference; import net.yacy.kelondro.rwi.Reference; diff --git a/source/net/yacy/kelondro/data/word/WordReferenceFactory.java b/source/net/yacy/kelondro/data/word/WordReferenceFactory.java index e29413ca54..a7db4c03a7 100644 --- a/source/net/yacy/kelondro/data/word/WordReferenceFactory.java +++ b/source/net/yacy/kelondro/data/word/WordReferenceFactory.java @@ -35,7 +35,7 @@ import java.util.SortedMap; import java.util.TreeMap; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.util.ByteBuffer; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row.Entry; diff --git a/source/net/yacy/kelondro/data/word/WordReferenceRow.java b/source/net/yacy/kelondro/data/word/WordReferenceRow.java index 74333e4987..45614a3a82 100644 --- a/source/net/yacy/kelondro/data/word/WordReferenceRow.java +++ b/source/net/yacy/kelondro/data/word/WordReferenceRow.java @@ -30,7 +30,7 @@ import java.util.Collection; import net.yacy.cora.date.MicroDate; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.order.Base64Order; import net.yacy.cora.util.ByteArray; import net.yacy.kelondro.index.Column; diff --git a/source/net/yacy/kelondro/data/word/WordReferenceVars.java b/source/net/yacy/kelondro/data/word/WordReferenceVars.java index 8ecb4c17bf..8cc47ecd9b 100644 --- a/source/net/yacy/kelondro/data/word/WordReferenceVars.java +++ b/source/net/yacy/kelondro/data/word/WordReferenceVars.java @@ -33,9 +33,9 @@ import java.util.concurrent.LinkedBlockingQueue; import net.yacy.cora.date.MicroDate; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.order.Base64Order; import net.yacy.cora.util.ByteArray; import net.yacy.cora.util.ConcurrentLog; @@ -83,7 +83,7 @@ public WordReferenceVars(final URIMetadataRow md, final boolean local) { this.positions = new LinkedBlockingQueue(); this.positions.add(1); String urlNormalform = md.url().toNormalform(true); - this.urlcomps = MultiProtocolURI.urlComps(urlNormalform).length; + this.urlcomps = MultiProtocolURL.urlComps(urlNormalform).length; this.urllength = urlNormalform.length(); this.virtualAge = -1; // compute that later // the following fields cannot be computed here very easy and are just filled with dummy values diff --git a/source/net/yacy/kelondro/index/IndexTest.java b/source/net/yacy/kelondro/index/IndexTest.java index 8d180e87e9..375b05b04c 100644 --- a/source/net/yacy/kelondro/index/IndexTest.java +++ b/source/net/yacy/kelondro/index/IndexTest.java @@ -32,8 +32,8 @@ import java.util.Random; import java.util.TreeMap; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.storage.HandleMap; import net.yacy.cora.util.ByteArray; diff --git a/source/net/yacy/kelondro/index/Row.java b/source/net/yacy/kelondro/index/Row.java index 391b0b3a5e..85240805ce 100644 --- a/source/net/yacy/kelondro/index/Row.java +++ b/source/net/yacy/kelondro/index/Row.java @@ -37,8 +37,8 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ConcurrentHashMap; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.AbstractOrder; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.ByteOrder; diff --git a/source/net/yacy/kelondro/index/RowCollection.java b/source/net/yacy/kelondro/index/RowCollection.java index 056bc2b027..442d47c2e0 100644 --- a/source/net/yacy/kelondro/index/RowCollection.java +++ b/source/net/yacy/kelondro/index/RowCollection.java @@ -33,8 +33,8 @@ import java.util.Random; import java.util.concurrent.Callable; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.order.NaturalOrder; diff --git a/source/net/yacy/kelondro/index/RowHandleSet.java b/source/net/yacy/kelondro/index/RowHandleSet.java index 7c9fe34cfa..49b5038cea 100644 --- a/source/net/yacy/kelondro/index/RowHandleSet.java +++ b/source/net/yacy/kelondro/index/RowHandleSet.java @@ -38,7 +38,7 @@ import java.util.Iterator; import java.util.Set; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.order.NaturalOrder; diff --git a/source/net/yacy/kelondro/index/RowSet.java b/source/net/yacy/kelondro/index/RowSet.java index 43b7e152ac..d43ef65f1e 100644 --- a/source/net/yacy/kelondro/index/RowSet.java +++ b/source/net/yacy/kelondro/index/RowSet.java @@ -34,8 +34,8 @@ import java.util.Random; import java.util.TreeMap; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.order.NaturalOrder; diff --git a/source/net/yacy/kelondro/io/AbstractWriter.java b/source/net/yacy/kelondro/io/AbstractWriter.java index 2ad3a8ce5d..f0da9b9597 100644 --- a/source/net/yacy/kelondro/io/AbstractWriter.java +++ b/source/net/yacy/kelondro/io/AbstractWriter.java @@ -32,7 +32,7 @@ import java.util.Iterator; import java.util.Map; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.util.ByteBuffer; diff --git a/source/net/yacy/kelondro/io/BufferedRecords.java b/source/net/yacy/kelondro/io/BufferedRecords.java index c98e2b4174..68656adcef 100644 --- a/source/net/yacy/kelondro/io/BufferedRecords.java +++ b/source/net/yacy/kelondro/io/BufferedRecords.java @@ -29,7 +29,7 @@ import java.util.Map; import java.util.TreeMap; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.util.ConcurrentLog; import net.yacy.kelondro.util.FileUtils; diff --git a/source/net/yacy/kelondro/io/CachedRecords.java b/source/net/yacy/kelondro/io/CachedRecords.java index 70366d0a0e..6b98a73320 100644 --- a/source/net/yacy/kelondro/io/CachedRecords.java +++ b/source/net/yacy/kelondro/io/CachedRecords.java @@ -30,7 +30,7 @@ import java.io.IOException; import java.io.RandomAccessFile; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.util.ConcurrentLog; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.MemoryControl; diff --git a/source/net/yacy/kelondro/io/CharBuffer.java b/source/net/yacy/kelondro/io/CharBuffer.java index 401e2da83b..5edc4b54a4 100644 --- a/source/net/yacy/kelondro/io/CharBuffer.java +++ b/source/net/yacy/kelondro/io/CharBuffer.java @@ -31,7 +31,7 @@ import java.io.Writer; import java.util.Properties; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; public final class CharBuffer extends Writer { diff --git a/source/net/yacy/kelondro/io/Records.java b/source/net/yacy/kelondro/io/Records.java index 0f3d309595..02d139f92b 100644 --- a/source/net/yacy/kelondro/io/Records.java +++ b/source/net/yacy/kelondro/io/Records.java @@ -30,7 +30,7 @@ import java.io.IOException; import java.io.RandomAccessFile; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.util.ConcurrentLog; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.MemoryControl; diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainer.java b/source/net/yacy/kelondro/rwi/ReferenceContainer.java index def34d070d..d33cdec498 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainer.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainer.java @@ -35,7 +35,7 @@ import java.util.List; import java.util.TreeMap; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.ByteOrder; import net.yacy.cora.storage.HandleSet; diff --git a/source/net/yacy/kelondro/rwi/ReferenceIterator.java b/source/net/yacy/kelondro/rwi/ReferenceIterator.java index 4ce6e0e7a0..6acb1eeff8 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceIterator.java +++ b/source/net/yacy/kelondro/rwi/ReferenceIterator.java @@ -30,7 +30,7 @@ import java.io.IOException; import java.util.Map; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.LookAheadIterator; diff --git a/source/net/yacy/kelondro/table/Relations.java b/source/net/yacy/kelondro/table/Relations.java index 3a5d5ec2a2..da2116c48a 100644 --- a/source/net/yacy/kelondro/table/Relations.java +++ b/source/net/yacy/kelondro/table/Relations.java @@ -30,7 +30,7 @@ import java.io.IOException; import java.util.HashMap; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.NaturalOrder; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.NumberTools; diff --git a/source/net/yacy/kelondro/table/SQLTable.java b/source/net/yacy/kelondro/table/SQLTable.java index fa2f947aec..df8b9a5b65 100644 --- a/source/net/yacy/kelondro/table/SQLTable.java +++ b/source/net/yacy/kelondro/table/SQLTable.java @@ -39,8 +39,8 @@ import java.util.Map; import java.util.TreeMap; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.util.ConcurrentLog; import net.yacy.kelondro.index.Index; diff --git a/source/net/yacy/kelondro/table/Table.java b/source/net/yacy/kelondro/table/Table.java index e0faf9577c..fb2d9ef7d7 100644 --- a/source/net/yacy/kelondro/table/Table.java +++ b/source/net/yacy/kelondro/table/Table.java @@ -38,7 +38,7 @@ import java.util.TreeMap; import java.util.TreeSet; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.order.CloneableIterator; import net.yacy.cora.order.NaturalOrder; import net.yacy.cora.storage.HandleMap; diff --git a/source/net/yacy/kelondro/util/BDecoder.java b/source/net/yacy/kelondro/util/BDecoder.java index 4c86ae3105..d12b0fd307 100644 --- a/source/net/yacy/kelondro/util/BDecoder.java +++ b/source/net/yacy/kelondro/util/BDecoder.java @@ -36,8 +36,8 @@ import java.util.List; import java.util.Map; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; public class BDecoder { diff --git a/source/net/yacy/kelondro/util/BEncoder.java b/source/net/yacy/kelondro/util/BEncoder.java index 28ed02c255..7dd7060e83 100644 --- a/source/net/yacy/kelondro/util/BEncoder.java +++ b/source/net/yacy/kelondro/util/BEncoder.java @@ -29,7 +29,7 @@ import java.util.HashMap; import java.util.Map; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.kelondro.util.BDecoder.BObject; public class BEncoder { diff --git a/source/net/yacy/kelondro/util/FileUtils.java b/source/net/yacy/kelondro/util/FileUtils.java index ee210a8829..4a637491aa 100644 --- a/source/net/yacy/kelondro/util/FileUtils.java +++ b/source/net/yacy/kelondro/util/FileUtils.java @@ -53,7 +53,7 @@ import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.storage.Files; import net.yacy.cora.util.ConcurrentLog; diff --git a/source/net/yacy/kelondro/util/OS.java b/source/net/yacy/kelondro/util/OS.java index c600cd9c81..502d2e655d 100644 --- a/source/net/yacy/kelondro/util/OS.java +++ b/source/net/yacy/kelondro/util/OS.java @@ -33,7 +33,7 @@ import java.util.Properties; import java.util.Vector; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.NumberTools; import net.yacy.server.serverCore; diff --git a/source/net/yacy/kelondro/util/SetTools.java b/source/net/yacy/kelondro/util/SetTools.java index 9cd054b42c..3306e83721 100644 --- a/source/net/yacy/kelondro/util/SetTools.java +++ b/source/net/yacy/kelondro/util/SetTools.java @@ -43,7 +43,7 @@ import java.util.TreeMap; import java.util.TreeSet; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.storage.HandleSet; import net.yacy.cora.util.ConcurrentLog; diff --git a/source/net/yacy/peers/Accessible.java b/source/net/yacy/peers/Accessible.java index 15f47d1d07..c3209bb638 100644 --- a/source/net/yacy/peers/Accessible.java +++ b/source/net/yacy/peers/Accessible.java @@ -30,7 +30,7 @@ import java.io.File; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.kelondro.util.FileUtils; import net.yacy.search.Switchboard; diff --git a/source/net/yacy/peers/DHTSelection.java b/source/net/yacy/peers/DHTSelection.java index 268357814f..cfc05a1d72 100644 --- a/source/net/yacy/peers/DHTSelection.java +++ b/source/net/yacy/peers/DHTSelection.java @@ -36,7 +36,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.federate.yacy.Distribution; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.Digest; diff --git a/source/net/yacy/peers/Dispatcher.java b/source/net/yacy/peers/Dispatcher.java index b206ba3cc9..427549e88e 100644 --- a/source/net/yacy/peers/Dispatcher.java +++ b/source/net/yacy/peers/Dispatcher.java @@ -32,7 +32,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.federate.yacy.Distribution; import net.yacy.cora.order.Base64Order; import net.yacy.cora.storage.HandleSet; diff --git a/source/net/yacy/peers/EventChannel.java b/source/net/yacy/peers/EventChannel.java index c21a7d1512..25ac25f9bb 100644 --- a/source/net/yacy/peers/EventChannel.java +++ b/source/net/yacy/peers/EventChannel.java @@ -29,8 +29,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import net.yacy.cora.document.RSSFeed; -import net.yacy.cora.document.RSSMessage; +import net.yacy.cora.document.feed.RSSFeed; +import net.yacy.cora.document.feed.RSSMessage; public enum EventChannel { TEST, diff --git a/source/net/yacy/peers/Network.java b/source/net/yacy/peers/Network.java index 1d2b47f0b8..2b4935205e 100644 --- a/source/net/yacy/peers/Network.java +++ b/source/net/yacy/peers/Network.java @@ -50,12 +50,12 @@ import java.util.concurrent.Semaphore; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.RSSFeed; -import net.yacy.cora.document.RSSMessage; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.feed.RSSFeed; +import net.yacy.cora.document.feed.RSSMessage; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.Domains; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.peers.operation.yacySeedUploadFile; import net.yacy.peers.operation.yacySeedUploadFtp; import net.yacy.peers.operation.yacySeedUploadScp; @@ -753,7 +753,7 @@ public static final String saveSeedList(final Switchboard sb) { } // ensure that the seed file url is configured properly - DigestURI seedURL; + DigestURL seedURL; try { final String seedURLStr = sb.peers.mySeed().get(Seed.SEEDLISTURL, ""); if ( seedURLStr.isEmpty() ) { @@ -763,7 +763,7 @@ public static final String saveSeedList(final Switchboard sb) { "https://")) ) { throw new MalformedURLException("Unsupported protocol."); } - seedURL = new DigestURI(seedURLStr); + seedURL = new DigestURL(seedURLStr); final String host = seedURL.getHost(); if (Domains.isLocalhost(host)) { // check seedlist reacheable final String errorMsg = "seedURL in localhost rejected (localhost can't be reached from outside)"; diff --git a/source/net/yacy/peers/NewsDB.java b/source/net/yacy/peers/NewsDB.java index a01ac8a51e..cef2912a45 100644 --- a/source/net/yacy/peers/NewsDB.java +++ b/source/net/yacy/peers/NewsDB.java @@ -54,7 +54,7 @@ import java.util.Properties; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.NaturalOrder; import net.yacy.cora.util.ConcurrentLog; diff --git a/source/net/yacy/peers/NewsPool.java b/source/net/yacy/peers/NewsPool.java index 0be6eeac34..7d946359a5 100644 --- a/source/net/yacy/peers/NewsPool.java +++ b/source/net/yacy/peers/NewsPool.java @@ -53,9 +53,9 @@ import java.util.Properties; import java.util.Set; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; @@ -343,13 +343,13 @@ public void enqueueIncomingNews(final NewsDB.Record record) throws IOException, if (record.created().getTime() == 0) return; final Map attributes = record.attributes(); if (attributes.containsKey("url")){ - if (Switchboard.urlBlacklist.isListed(BlacklistType.NEWS, new DigestURI(attributes.get("url")))){ + if (Switchboard.urlBlacklist.isListed(BlacklistType.NEWS, new DigestURL(attributes.get("url")))){ System.out.println("DEBUG: ignored news-entry url blacklisted: " + attributes.get("url")); return; } } if (attributes.containsKey("startURL")){ - if (Switchboard.urlBlacklist.isListed(BlacklistType.NEWS, new DigestURI(attributes.get("startURL")))){ + if (Switchboard.urlBlacklist.isListed(BlacklistType.NEWS, new DigestURL(attributes.get("startURL")))){ System.out.println("DEBUG: ignored news-entry url blacklisted: " + attributes.get("startURL")); return; } diff --git a/source/net/yacy/peers/NewsQueue.java b/source/net/yacy/peers/NewsQueue.java index f06bc8f983..6d37ed7f9c 100644 --- a/source/net/yacy/peers/NewsQueue.java +++ b/source/net/yacy/peers/NewsQueue.java @@ -50,7 +50,7 @@ import java.util.Iterator; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.NaturalOrder; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; diff --git a/source/net/yacy/peers/PeerActions.java b/source/net/yacy/peers/PeerActions.java index 0ce06780bf..c7bc831b4d 100644 --- a/source/net/yacy/peers/PeerActions.java +++ b/source/net/yacy/peers/PeerActions.java @@ -26,8 +26,8 @@ import java.util.Map; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.RSSMessage; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.feed.RSSMessage; import net.yacy.cora.storage.ConcurrentARC; import net.yacy.kelondro.util.MapTools; import net.yacy.peers.operation.yacyVersion; diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index fe11e45128..d2732b95d1 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -59,17 +59,13 @@ import net.yacy.migration; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.JSONArray; -import net.yacy.cora.document.JSONException; -import net.yacy.cora.document.JSONObject; -import net.yacy.cora.document.JSONTokener; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.RSSFeed; -import net.yacy.cora.document.RSSMessage; -import net.yacy.cora.document.RSSReader; -import net.yacy.cora.document.UTF8; import net.yacy.cora.document.analysis.Classification; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.feed.RSSFeed; +import net.yacy.cora.document.feed.RSSMessage; +import net.yacy.cora.document.feed.RSSReader; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.opensearch.SRURSSConnector; import net.yacy.cora.federate.solr.connector.RemoteSolrConnector; import net.yacy.cora.federate.solr.connector.SolrConnector; @@ -85,6 +81,10 @@ import net.yacy.cora.storage.HandleSet; import net.yacy.cora.util.ByteBuffer; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.cora.util.JSONArray; +import net.yacy.cora.util.JSONException; +import net.yacy.cora.util.JSONObject; +import net.yacy.cora.util.JSONTokener; import net.yacy.cora.util.SpaceExceededException; import net.yacy.crawler.data.ResultURLs; import net.yacy.crawler.data.ResultURLs.EventOrigin; @@ -154,7 +154,7 @@ private static byte[] postToFile( final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent); httpClient.setTimout(timeout); return httpClient.POSTbytes( - new MultiProtocolURI("http://" + targetAddress + "/yacy/" + filename), + new MultiProtocolURL("http://" + targetAddress + "/yacy/" + filename), Seed.b64Hash2hexHash(targetPeerHash) + ".yacyh", parts, false); @@ -197,7 +197,7 @@ public static int hello( final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 30000); content = httpClient.POSTbytes( - new MultiProtocolURI("http://" + address + "/yacy/hello.html"), + new MultiProtocolURL("http://" + address + "/yacy/hello.html"), Seed.b64Hash2hexHash(otherHash) + ".yacyh", parts, false); @@ -517,7 +517,7 @@ public static RSSFeed queryRemoteCrawlURLs( // final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), (int) maxTime, target.getHexHash() + ".yacyh", parts); final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, (int) maxTime); final byte[] result = - httpClient.POSTbytes(new MultiProtocolURI("http://" + httpClient.POSTbytes(new MultiProtocolURL("http://" + target.getClusterAddress() + "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false); final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result); @@ -938,7 +938,7 @@ public SearchResult( } final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 8000); - byte[] a = httpClient.POSTbytes(new MultiProtocolURI("http://" + hostaddress + "/yacy/search.html"), hostname, parts, false); + byte[] a = httpClient.POSTbytes(new MultiProtocolURL("http://" + hostaddress + "/yacy/search.html"), hostname, parts, false); if (a != null && a.length > 200000) { // there is something wrong. This is too large, maybe a hack on the other side? a = null; @@ -1294,7 +1294,7 @@ public static Map crawlReceipt( final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 10000); final byte[] content = httpClient.POSTbytes( - new MultiProtocolURI("http://" + address + "/yacy/crawlReceipt.html"), + new MultiProtocolURL("http://" + address + "/yacy/crawlReceipt.html"), target.getHexHash() + ".yacyh", parts, false); @@ -1473,7 +1473,7 @@ private static Map transferRWI( final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout); final byte[] content = httpClient.POSTbytes( - new MultiProtocolURI("http://" + address + "/yacy/transferRWI.html"), + new MultiProtocolURL("http://" + address + "/yacy/transferRWI.html"), targetSeed.getHexHash() + ".yacyh", parts, gzipBody); @@ -1531,7 +1531,7 @@ private static Map transferURL( final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, timeout); final byte[] content = httpClient.POSTbytes( - new MultiProtocolURI("http://" + address + "/yacy/transferURL.html"), + new MultiProtocolURL("http://" + address + "/yacy/transferURL.html"), targetSeed.getHexHash() + ".yacyh", parts, gzipBody); @@ -1564,7 +1564,7 @@ public static Map getProfile(final Seed targetSeed) { final HTTPClient httpclient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 15000); final byte[] content = httpclient.POSTbytes( - new MultiProtocolURI("http://" + address + "/yacy/profile.html"), + new MultiProtocolURL("http://" + address + "/yacy/profile.html"), targetSeed.getHexHash() + ".yacyh", parts, false); diff --git a/source/net/yacy/peers/RemoteSearch.java b/source/net/yacy/peers/RemoteSearch.java index 73e468d27b..53685298ea 100644 --- a/source/net/yacy/peers/RemoteSearch.java +++ b/source/net/yacy/peers/RemoteSearch.java @@ -32,7 +32,7 @@ import org.apache.solr.client.solrj.SolrQuery; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.storage.HandleSet; import net.yacy.cora.util.ConcurrentLog; import net.yacy.repository.Blacklist; diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java index 0765a14f2b..82e45db1e9 100644 --- a/source/net/yacy/peers/Seed.java +++ b/source/net/yacy/peers/Seed.java @@ -60,8 +60,8 @@ import net.yacy.cora.date.AbstractFormatter; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.federate.yacy.Distribution; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.Digest; diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java index 06d7e9f073..85cd90f68f 100644 --- a/source/net/yacy/peers/SeedDB.java +++ b/source/net/yacy/peers/SeedDB.java @@ -38,8 +38,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.Distribution; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.ClientIdentification; @@ -50,7 +51,6 @@ import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; import net.yacy.kelondro.blob.MapDataMining; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.kelondroException; @@ -760,7 +760,7 @@ private ArrayList storeSeedList(final File seedFile, final boolean addMy protected String uploadSeedList(final yacySeedUploader uploader, final serverSwitch sb, final SeedDB seedDB, - final DigestURI seedURL) throws Exception { + final DigestURL seedURL) throws Exception { // upload a seed file, if possible if (seedURL == null) throw new NullPointerException("UPLOAD - Error: URL not given"); @@ -801,7 +801,7 @@ protected String uploadSeedList(final yacySeedUploader uploader, return log; } - private static Iterator downloadSeedFile(final DigestURI seedURL) throws IOException { + private static Iterator downloadSeedFile(final DigestURL seedURL) throws IOException { // Configure http headers final RequestHeader reqHeader = new RequestHeader(); reqHeader.put(HeaderFramework.PRAGMA, "no-cache"); diff --git a/source/net/yacy/peers/Transmission.java b/source/net/yacy/peers/Transmission.java index 9beafd8c33..47799fd7cb 100644 --- a/source/net/yacy/peers/Transmission.java +++ b/source/net/yacy/peers/Transmission.java @@ -32,7 +32,7 @@ import java.util.Random; import java.util.Set; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.storage.HandleSet; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; diff --git a/source/net/yacy/peers/graphics/NetworkGraph.java b/source/net/yacy/peers/graphics/NetworkGraph.java index e60301e272..ef9e2f4951 100644 --- a/source/net/yacy/peers/graphics/NetworkGraph.java +++ b/source/net/yacy/peers/graphics/NetworkGraph.java @@ -34,9 +34,9 @@ import java.util.Iterator; import java.util.List; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.Hit; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.feed.Hit; import net.yacy.cora.federate.yacy.Distribution; import net.yacy.cora.util.ConcurrentLog; import net.yacy.peers.EventChannel; diff --git a/source/net/yacy/peers/graphics/OSMTile.java b/source/net/yacy/peers/graphics/OSMTile.java index b120ecd485..49381b77a3 100644 --- a/source/net/yacy/peers/graphics/OSMTile.java +++ b/source/net/yacy/peers/graphics/OSMTile.java @@ -34,12 +34,12 @@ import javax.imageio.ImageIO; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.util.ConcurrentLog; import net.yacy.crawler.data.Cache; import net.yacy.crawler.retrieval.Response; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.Switchboard; import net.yacy.visualization.RasterPlotter; @@ -101,9 +101,9 @@ public void run() { } public static BufferedImage getSingleTile(final tileCoordinates tile, final int retry) { - DigestURI tileURL; + DigestURL tileURL; try { - tileURL = new DigestURI(tile.url(retry)); + tileURL = new DigestURL(tile.url(retry)); } catch (final MalformedURLException e) { return null; } diff --git a/source/net/yacy/peers/graphics/WebStructureGraph.java b/source/net/yacy/peers/graphics/WebStructureGraph.java index aa7a23c471..8dc512ce5a 100644 --- a/source/net/yacy/peers/graphics/WebStructureGraph.java +++ b/source/net/yacy/peers/graphics/WebStructureGraph.java @@ -45,8 +45,9 @@ import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.date.MicroDate; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.order.Base64Order; import net.yacy.cora.sorting.ClusteredScoreMap; import net.yacy.cora.sorting.ReversibleScoreMap; @@ -54,7 +55,6 @@ import net.yacy.cora.util.LookAheadIterator; import net.yacy.cora.util.SpaceExceededException; import net.yacy.document.Document; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.index.Row; import net.yacy.kelondro.index.Row.Entry; import net.yacy.kelondro.rwi.AbstractReference; @@ -81,10 +81,10 @@ public class WebStructureGraph { private final static LearnObject leanrefObjectPOISON = new LearnObject(null, null); private static class LearnObject { - private final DigestURI url; - private final Set globalRefURLs; + private final DigestURL url; + private final Set globalRefURLs; - private LearnObject(final DigestURI url, final Set globalRefURLs) { + private LearnObject(final DigestURL url, final Set globalRefURLs) { this.url = url; this.globalRefURLs = globalRefURLs; } @@ -157,13 +157,13 @@ public void clear() { this.structure_new.clear(); } - public void generateCitationReference(final DigestURI url, final Document document) { + public void generateCitationReference(final DigestURL url, final Document document) { // generate citation reference - final Map hl = document.getHyperlinks(); - final Iterator it = hl.keySet().iterator(); - final HashSet globalRefURLs = new HashSet(); + final Map hl = document.getHyperlinks(); + final Iterator it = hl.keySet().iterator(); + final HashSet globalRefURLs = new HashSet(); final String refhost = url.getHost(); - DigestURI u; + DigestURL u; int maxref = 1000; while ( it.hasNext() && maxref-- > 0 ) { u = it.next(); @@ -189,8 +189,8 @@ public void generateCitationReference(final DigestURI url, final Document docume } } - public void generateCitationReference(final DigestURI from, final DigestURI to) { - final HashSet globalRefURLs = new HashSet(); + public void generateCitationReference(final DigestURL from, final DigestURL to) { + final HashSet globalRefURLs = new HashSet(); final String refhost = from.getHost(); if (refhost != null && to.getHost() != null && !to.getHost().equals(refhost)) globalRefURLs.add(to); final LearnObject lro = new LearnObject(from, globalRefURLs); @@ -586,7 +586,7 @@ public String hostHash2hostName(final String hosthash) { private void learnrefs(final LearnObject lro) { final Set refhosts = new HashSet(); String hosthash; - for ( final DigestURI u : lro.globalRefURLs ) { + for ( final DigestURL u : lro.globalRefURLs ) { if (Switchboard.getSwitchboard().shallTerminate()) break; hosthash = ASCII.String(u.hash(), 6, 6); if (!exists(hosthash)) { @@ -597,7 +597,7 @@ private void learnrefs(final LearnObject lro) { } refhosts.add(hosthash); } - final DigestURI url = lro.url; + final DigestURL url = lro.url; hosthash = ASCII.String(url.hash(), 6, 6); // parse the new reference string and join it with the stored references diff --git a/source/net/yacy/peers/operation/yacyRelease.java b/source/net/yacy/peers/operation/yacyRelease.java index 703b1eb111..95f0c4a353 100644 --- a/source/net/yacy/peers/operation/yacyRelease.java +++ b/source/net/yacy/peers/operation/yacyRelease.java @@ -37,15 +37,17 @@ import java.security.PublicKey; import java.security.SignatureException; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.AnchorURL; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.ClientIdentification; @@ -55,7 +57,6 @@ import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.Document; import net.yacy.document.parser.tarParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.io.CharBuffer; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.OS; @@ -75,17 +76,17 @@ public final class yacyRelease extends yacyVersion { public final static List latestReleaseLocations = new ArrayList(); // will be initialized with value in defaults/yacy.network.freeworld.unit public static String startParameter = ""; - private MultiProtocolURI url; + private MultiProtocolURL url; private File releaseFile; private PublicKey publicKey; - public yacyRelease(final MultiProtocolURI url) { + public yacyRelease(final MultiProtocolURL url) { super(url.getFileName(), url.getHost()); this.url = url; } - private yacyRelease(final MultiProtocolURI url, final PublicKey publicKey) { + private yacyRelease(final MultiProtocolURL url, final PublicKey publicKey) { this(url); this.publicKey = publicKey; } @@ -95,7 +96,7 @@ public yacyRelease(final File releaseFile) { this.releaseFile = releaseFile; } - public MultiProtocolURI getUrl() { + public MultiProtocolURL getUrl() { return this.url; } @@ -236,7 +237,7 @@ private static DevAndMainVersions allReleaseFrom(final yacyUpdateLocation locati // returns the version info if successful, null otherwise Document scraper; try { - final DigestURI uri = location.getLocationURL(); + final DigestURL uri = location.getLocationURL(); Thread.currentThread().setName("allReleaseFrom - host " + uri.getHost()); // makes it more easy to see which release blocks process in thread dump scraper = Switchboard.getSwitchboard().loader.loadDocument(uri, CacheStrategy.NOCACHE, null, ClientIdentification.yacyInternetCrawlerAgent); } catch (final IOException e) { @@ -244,10 +245,10 @@ private static DevAndMainVersions allReleaseFrom(final yacyUpdateLocation locati } // analyze links in scraper resource, and find link to latest release in it - final Map anchors = scraper.getAnchors(); // a url (String) / name (String) relation + final Collection anchors = scraper.getAnchors(); // a url (String) / name (String) relation final TreeSet mainReleases = new TreeSet(); final TreeSet devReleases = new TreeSet(); - for (final DigestURI url : anchors.keySet()) { + for (final DigestURL url : anchors) { try { final yacyRelease release = new yacyRelease(url, location.getPublicKey()); //System.out.println("r " + release.toAnchor()); diff --git a/source/net/yacy/peers/operation/yacySeedUploadScp.java b/source/net/yacy/peers/operation/yacySeedUploadScp.java index bb8ba77fcf..f4c086e5a3 100644 --- a/source/net/yacy/peers/operation/yacySeedUploadScp.java +++ b/source/net/yacy/peers/operation/yacySeedUploadScp.java @@ -32,7 +32,7 @@ import java.io.InputStream; import java.io.OutputStream; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.server.serverSwitch; import com.jcraft.jsch.Channel; diff --git a/source/net/yacy/peers/operation/yacyUpdateLocation.java b/source/net/yacy/peers/operation/yacyUpdateLocation.java index 5e2c62bee3..4e3cad9076 100644 --- a/source/net/yacy/peers/operation/yacyUpdateLocation.java +++ b/source/net/yacy/peers/operation/yacyUpdateLocation.java @@ -29,7 +29,7 @@ import java.security.PublicKey; -import net.yacy.kelondro.data.meta.DigestURI; +import net.yacy.cora.document.id.DigestURL; /** @@ -37,15 +37,15 @@ * */ public class yacyUpdateLocation { - private final DigestURI locationURL; + private final DigestURL locationURL; private final PublicKey publicKey; - public yacyUpdateLocation(DigestURI locationURL, PublicKey publicKey) { + public yacyUpdateLocation(DigestURL locationURL, PublicKey publicKey) { this.locationURL = locationURL; this.publicKey = publicKey; } - public DigestURI getLocationURL() { + public DigestURL getLocationURL() { return this.locationURL; } public PublicKey getPublicKey() { diff --git a/source/net/yacy/repository/Blacklist.java b/source/net/yacy/repository/Blacklist.java index 1129d2e1bf..e4105b89c6 100644 --- a/source/net/yacy/repository/Blacklist.java +++ b/source/net/yacy/repository/Blacklist.java @@ -45,11 +45,11 @@ import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.storage.HandleSet; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; import net.yacy.data.ListManager; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.index.RowHandleSet; @@ -405,7 +405,7 @@ public final boolean isListed(final BlacklistType blacklistType, final URIMetada * @param url Entry to be checked * @return Whether the given entry is blacklisted */ - public final boolean isListed(final BlacklistType blacklistType, final DigestURI url) { + public final boolean isListed(final BlacklistType blacklistType, final DigestURL url) { if (url == null) { throw new IllegalArgumentException("url may not be null"); } diff --git a/source/net/yacy/repository/FilterEngine.java b/source/net/yacy/repository/FilterEngine.java index be34989f51..51037b6a21 100644 --- a/source/net/yacy/repository/FilterEngine.java +++ b/source/net/yacy/repository/FilterEngine.java @@ -11,9 +11,9 @@ import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.storage.HashARC; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.data.meta.DigestURI; /** * a URL filter engine for black and white lists @@ -51,7 +51,7 @@ public int compareTo(final FilterEntry fe) { } } - protected HashARC> cachedUrlHashs = null; + protected HashARC> cachedUrlHashs = null; protected Map> hostpaths_matchable = null; protected Map> hostpaths_notmatchable = null; @@ -60,7 +60,7 @@ public FilterEngine() { // prepare the data structure this.hostpaths_matchable = new HashMap>(); this.hostpaths_notmatchable = new HashMap>(); - this.cachedUrlHashs = new HashARC>(CACHE_SIZE); + this.cachedUrlHashs = new HashARC>(CACHE_SIZE); } public void clear() { @@ -112,7 +112,7 @@ public void removeAll(final String host) { this.hostpaths_notmatchable.remove(host); } - public boolean isListed(final DigestURI url, final EnumSet type) { + public boolean isListed(final DigestURL url, final EnumSet type) { // trival anwser if (url.getHost() == null) return false; diff --git a/source/net/yacy/repository/LoaderDispatcher.java b/source/net/yacy/repository/LoaderDispatcher.java index e92af6cef4..776383dc3f 100644 --- a/source/net/yacy/repository/LoaderDispatcher.java +++ b/source/net/yacy/repository/LoaderDispatcher.java @@ -38,8 +38,9 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.HeaderFramework; @@ -58,7 +59,6 @@ import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.TextParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; @@ -74,7 +74,7 @@ public final class LoaderDispatcher { private final FTPLoader ftpLoader; private final SMBLoader smbLoader; private final FileLoader fileLoader; - private final ConcurrentHashMap loaderSteering; // a map that delivers a 'finish' semaphore for urls + private final ConcurrentHashMap loaderSteering; // a map that delivers a 'finish' semaphore for urls private final ConcurrentLog log; public LoaderDispatcher(final Switchboard sb) { @@ -87,7 +87,7 @@ public LoaderDispatcher(final Switchboard sb) { this.ftpLoader = new FTPLoader(sb, this.log); this.smbLoader = new SMBLoader(sb, this.log); this.fileLoader = new FileLoader(sb, this.log); - this.loaderSteering = new ConcurrentHashMap(); + this.loaderSteering = new ConcurrentHashMap(); } public boolean isSupportedProtocol(final String protocol) { @@ -108,7 +108,7 @@ public HashSet getSupportedProtocols() { * @return the request object */ public Request request( - final DigestURI url, + final DigestURL url, final boolean forText, final boolean global ) { @@ -132,7 +132,7 @@ public Request request( 0); } - public void load(final DigestURI url, final CacheStrategy cacheStratgy, final int maxFileSize, final File targetFile, BlacklistType blacklistType, ClientIdentification.Agent agent) throws IOException { + public void load(final DigestURL url, final CacheStrategy cacheStratgy, final int maxFileSize, final File targetFile, BlacklistType blacklistType, ClientIdentification.Agent agent) throws IOException { final byte[] b = load(request(url, false, true), cacheStratgy, maxFileSize, blacklistType, agent).getContent(); if (b == null) throw new IOException("load == null"); @@ -182,7 +182,7 @@ public Response load(final Request request, final CacheStrategy cacheStrategy, f */ private Response loadInternal(final Request request, CacheStrategy cacheStrategy, final int maxFileSize, final BlacklistType blacklistType, ClientIdentification.Agent agent) throws IOException { // get the protocol of the next URL - final DigestURI url = request.url(); + final DigestURL url = request.url(); if (url.isFile() || url.isSMB()) cacheStrategy = CacheStrategy.NOCACHE; // load just from the file system final String protocol = url.getProtocol(); final String host = url.getHost(); @@ -207,7 +207,7 @@ private Response loadInternal(final Request request, CacheStrategy cacheStrategy // in case that we want to return the cached content in the next step final RequestHeader requestHeader = new RequestHeader(); requestHeader.put(HeaderFramework.USER_AGENT, agent.userAgent); - DigestURI refererURL = null; + DigestURL refererURL = null; if (request.referrerhash() != null) refererURL = this.sb.getURL(request.referrerhash()); if (refererURL != null) requestHeader.put(RequestHeader.REFERER, refererURL.toNormalform(true)); final Response response = new Response( @@ -317,7 +317,7 @@ private Response loadInternal(final Request request, CacheStrategy cacheStrategy return response; } - private int protocolMaxFileSize(final DigestURI url) { + private int protocolMaxFileSize(final DigestURL url) { if (url.isHTTP() || url.isHTTPS()) return this.sb.getConfigInt("crawler.http.maxFileSize", HTTPLoader.DEFAULT_MAXFILESIZE); if (url.isFTP()) @@ -348,7 +348,7 @@ public Document[] loadDocuments(final Request request, final CacheStrategy cache // load resource final Response response = load(request, cacheStrategy, maxFileSize, blacklistType, agent); - final DigestURI url = request.url(); + final DigestURL url = request.url(); if (response == null) throw new IOException("no Response for url " + url); // if it is still not available, report an error @@ -358,11 +358,11 @@ public Document[] loadDocuments(final Request request, final CacheStrategy cache return response.parse(); } - public Document loadDocument(final DigestURI location, final CacheStrategy cachePolicy, BlacklistType blacklistType, final ClientIdentification.Agent agent) throws IOException { + public Document loadDocument(final DigestURL location, final CacheStrategy cachePolicy, BlacklistType blacklistType, final ClientIdentification.Agent agent) throws IOException { // load resource Request request = request(location, true, false); final Response response = this.load(request, cachePolicy, blacklistType, agent); - final DigestURI url = request.url(); + final DigestURL url = request.url(); if (response == null) throw new IOException("no Response for url " + url); // if it is still not available, report an error @@ -384,7 +384,7 @@ public Document loadDocument(final DigestURI location, final CacheStrategy cache * @return a map from URLs to the anchor texts of the urls * @throws IOException */ - public final Map loadLinks(final DigestURI url, final CacheStrategy cacheStrategy, BlacklistType blacklistType, final ClientIdentification.Agent agent) throws IOException { + public final Map loadLinks(final DigestURL url, final CacheStrategy cacheStrategy, BlacklistType blacklistType, final ClientIdentification.Agent agent) throws IOException { final Response response = load(request(url, true, false), cacheStrategy, Integer.MAX_VALUE, blacklistType, agent); if (response == null) throw new IOException("response == null"); final ResponseHeader responseHeader = response.getResponseHeader(); @@ -414,24 +414,24 @@ public synchronized static void cleanupAccessTimeTable(final long timeout) { } } - public void loadIfNotExistBackground(final DigestURI url, final File cache, final int maxFileSize, BlacklistType blacklistType, final ClientIdentification.Agent agent) { + public void loadIfNotExistBackground(final DigestURL url, final File cache, final int maxFileSize, BlacklistType blacklistType, final ClientIdentification.Agent agent) { new Loader(url, cache, maxFileSize, CacheStrategy.IFEXIST, blacklistType, agent).start(); } - public void loadIfNotExistBackground(final DigestURI url, final int maxFileSize, BlacklistType blacklistType, final ClientIdentification.Agent agent) { + public void loadIfNotExistBackground(final DigestURL url, final int maxFileSize, BlacklistType blacklistType, final ClientIdentification.Agent agent) { new Loader(url, null, maxFileSize, CacheStrategy.IFEXIST, blacklistType, agent).start(); } private class Loader extends Thread { - private final DigestURI url; + private final DigestURL url; private final File cache; private final int maxFileSize; private final CacheStrategy cacheStrategy; private final BlacklistType blacklistType; private final ClientIdentification.Agent agent; - public Loader(final DigestURI url, final File cache, final int maxFileSize, final CacheStrategy cacheStrategy, BlacklistType blacklistType, final ClientIdentification.Agent agent) { + public Loader(final DigestURL url, final File cache, final int maxFileSize, final CacheStrategy cacheStrategy, BlacklistType blacklistType, final ClientIdentification.Agent agent) { this.url = url; this.cache = cache; this.maxFileSize = maxFileSize; diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 2fe0affdfa..bb2a8b74c9 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -87,14 +87,15 @@ import net.yacy.contentcontrol.ContentControlFilterUpdateThread; import net.yacy.contentcontrol.SMWListSyncThread; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.RSSFeed; -import net.yacy.cora.document.RSSMessage; -import net.yacy.cora.document.RSSReader; -import net.yacy.cora.document.UTF8; import net.yacy.cora.document.WordCache; import net.yacy.cora.document.analysis.Classification; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.feed.RSSFeed; +import net.yacy.cora.document.feed.RSSMessage; +import net.yacy.cora.document.feed.RSSReader; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.solr.Ranking; import net.yacy.cora.federate.solr.SchemaConfiguration; import net.yacy.cora.federate.solr.instance.RemoteInstance; @@ -155,7 +156,6 @@ import net.yacy.document.parser.html.Evaluation; import net.yacy.gui.Tray; import net.yacy.kelondro.blob.Tables; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.rwi.ReferenceContainer; @@ -376,7 +376,7 @@ public void run() { // init sessionid name file final String sessionidNamesFile = getConfig("sessionidNamesFile", "defaults/sessionid.names"); this.log.config("Loading sessionid file " + sessionidNamesFile); - MultiProtocolURI.initSessionIDNames(FileUtils.loadList(new File(getAppPath(), sessionidNamesFile))); + MultiProtocolURL.initSessionIDNames(FileUtils.loadList(new File(getAppPath(), sessionidNamesFile))); // init tables this.tables = new WorkTables(this.workPath); @@ -1177,10 +1177,10 @@ public void overwriteNetworkDefinition() throws FileNotFoundException, IOExcepti if ( location.isEmpty() ) { break; } - DigestURI locationURL; + DigestURL locationURL; try { // try to parse url - locationURL = new DigestURI(location); + locationURL = new DigestURL(location); } catch (final MalformedURLException e ) { break; } @@ -1588,10 +1588,10 @@ public void urlRemove(final Segment segment, final byte[] hash) { this.crawlQueues.removeURL(hash); } - public DigestURI getURL(final byte[] urlhash) { + public DigestURL getURL(final byte[] urlhash) { if (urlhash == null) return null; if (urlhash.length == 0) return null; - final DigestURI url = this.index.fulltext().getURL(urlhash); + final DigestURL url = this.index.fulltext().getURL(urlhash); if (url != null) return url; return this.crawlQueues.getURL(urlhash); } @@ -1788,7 +1788,7 @@ public String toIndexer(final Response response) { // in the noIndexReason is set, indexing is not allowed if ( noIndexReason != null ) { // log cause and close queue - final DigestURI referrerURL = response.referrerURL(); + final DigestURL referrerURL = response.referrerURL(); //if (log.isFine()) log.logFine("deQueue: not indexed any word in URL " + response.url() + "; cause: " + noIndexReason); addURLtoErrorDB( response.url(), @@ -2511,7 +2511,7 @@ private Document[] parseDocument(final Response response) throws InterruptedExce ) ) { // get the hyperlinks - final Map hl = Document.getHyperlinks(documents); + final Map hl = Document.getHyperlinks(documents); boolean loadImages = getConfigBool("crawler.load.image", true); if (loadImages) hl.putAll(Document.getImagelinks(documents)); @@ -2524,8 +2524,8 @@ private Document[] parseDocument(final Response response) throws InterruptedExce } // insert those hyperlinks to the crawler - MultiProtocolURI nextUrl; - for ( final Map.Entry nextEntry : hl.entrySet() ) { + MultiProtocolURL nextUrl; + for ( final Map.Entry nextEntry : hl.entrySet() ) { // check for interruption checkInterruption(); @@ -2550,7 +2550,7 @@ private Document[] parseDocument(final Response response) throws InterruptedExce try { this.crawlStacker.enqueueEntry(new Request( response.initiator(), - new DigestURI(u), + new DigestURL(u), response.url().hash(), nextEntry.getValue(), new Date(), @@ -2697,8 +2697,8 @@ private void storeDocumentIndex( // CREATE INDEX final String dc_title = document.dc_title(); - final DigestURI url = document.dc_source(); - final DigestURI referrerURL = queueEntry.referrerURL(); + final DigestURL url = document.dc_source(); + final DigestURL referrerURL = queueEntry.referrerURL(); EventOrigin processCase = queueEntry.processCase(this.peers.mySeed().hash); CrawlProfile profile = queueEntry.profile(); @@ -2757,7 +2757,7 @@ private void storeDocumentIndex( feed.addMessage(new RSSMessage("Indexed web page", dc_title, queueEntry.url(), ASCII.String(queueEntry.url().hash()))); // store rss feeds in document into rss table - for ( final Map.Entry rssEntry : document.getRSS().entrySet() ) { + for ( final Map.Entry rssEntry : document.getRSS().entrySet() ) { final Tables.Data rssRow = new Tables.Data(); rssRow.put("referrer", url.hash()); rssRow.put("url", UTF8.getBytes(rssEntry.getKey().toNormalform(true))); @@ -2806,29 +2806,29 @@ private void storeDocumentIndex( } public final void addAllToIndex( - final DigestURI url, - final Map links, + final DigestURL url, + final Map links, final SearchEvent searchEvent, final String heuristicName, final Map collections) { - List urls = new ArrayList(); + List urls = new ArrayList(); // add the landing page to the index. should not load that again since it should be in the cache if (url != null) { urls.add(url); } // check if some of the links match with the query - final Map matcher = searchEvent.query.separateMatches(links); + final Map matcher = searchEvent.query.separateMatches(links); // take the matcher and load them all - for (final Map.Entry entry : matcher.entrySet()) { - urls.add(new DigestURI(entry.getKey(), (byte[]) null)); + for (final Map.Entry entry : matcher.entrySet()) { + urls.add(new DigestURL(entry.getKey(), (byte[]) null)); } // take then the no-matcher and load them also - for (final Map.Entry entry : links.entrySet()) { - urls.add(new DigestURI(entry.getKey(), (byte[]) null)); + for (final Map.Entry entry : links.entrySet()) { + urls.add(new DigestURL(entry.getKey(), (byte[]) null)); } addToIndex(urls, searchEvent, heuristicName, collections); } @@ -2845,11 +2845,11 @@ public void remove(final byte[] urlhash) { this.crawlQueues.removeURL(urlhash); } - public void stackURLs(Set rootURLs, final CrawlProfile profile, final Set successurls, final Map failurls) { + public void stackURLs(Set rootURLs, final CrawlProfile profile, final Set successurls, final Map failurls) { if (rootURLs == null || rootURLs.size() == 0) return; List stackthreads = new ArrayList(); // do this concurrently - for (DigestURI url: rootURLs) { - final DigestURI turl = url; + for (DigestURL url: rootURLs) { + final DigestURL turl = url; Thread t = new Thread() { public void run() { String failreason; @@ -2870,7 +2870,7 @@ public void run() { * @param url * @return null if this was ok. If this failed, return a string with a fail reason */ - public String stackUrl(CrawlProfile profile, DigestURI url) { + public String stackUrl(CrawlProfile profile, DigestURL url) { byte[] handle = ASCII.getBytes(profile.handle()); @@ -2986,15 +2986,15 @@ public String stackUrl(CrawlProfile profile, DigestURI url) { * @throws IOException * @throws Parser.Failure */ - public void addToIndex(final Collection urls, final SearchEvent searchEvent, final String heuristicName, final Map collections) { - Map urlmap = new HashMap(); - for (DigestURI url: urls) urlmap.put(ASCII.String(url.hash()), url); + public void addToIndex(final Collection urls, final SearchEvent searchEvent, final String heuristicName, final Map collections) { + Map urlmap = new HashMap(); + for (DigestURL url: urls) urlmap.put(ASCII.String(url.hash()), url); if (searchEvent != null) { for (String id: urlmap.keySet()) searchEvent.addHeuristic(ASCII.getBytes(id), heuristicName, true); } final Set existing = this.index.exists(urlmap.keySet()); final List requests = new ArrayList(); - for (Map.Entry e: urlmap.entrySet()) { + for (Map.Entry e: urlmap.entrySet()) { final String urlName = e.getValue().toNormalform(true); if (existing.contains(e.getKey())) { this.log.info("addToIndex: double " + urlName); @@ -3014,7 +3014,7 @@ public void addToIndex(final Collection urls, final SearchEvent searc @Override public void run() { for (Request request: requests) { - DigestURI url = request.url(); + DigestURL url = request.url(); String urlName = url.toNormalform(true); Thread.currentThread().setName("Switchboard.addToIndex:" + urlName); try { @@ -3066,13 +3066,13 @@ public void run() { * @param url the url that shall be indexed * @param asglobal true adds the url to global crawl queue (for remote crawling), false to the local crawler */ - public void addToCrawler(final Collection urls, final boolean asglobal) { - Map urlmap = new HashMap(); - for (DigestURI url: urls) urlmap.put(ASCII.String(url.hash()), url); + public void addToCrawler(final Collection urls, final boolean asglobal) { + Map urlmap = new HashMap(); + for (DigestURL url: urls) urlmap.put(ASCII.String(url.hash()), url); Set existingids = this.index.exists(urlmap.keySet()); - for (Map.Entry e: urlmap.entrySet()) { + for (Map.Entry e: urlmap.entrySet()) { if (existingids.contains(e.getKey())) continue; // double - DigestURI url = e.getValue(); + DigestURL url = e.getValue(); final Request request = this.loader.request(url, true, true); final CrawlProfile profile = this.crawler.getActive(ASCII.getBytes(request.profileHandle())); final String acceptedError = this.crawlStacker.checkAcceptance(url, profile, 0); @@ -3373,7 +3373,7 @@ public boolean dhtTransferJob() { } private void addURLtoErrorDB( - final DigestURI url, + final DigestURL url, final CrawlProfile profile, final byte[] referrerHash, final byte[] initiator, @@ -3408,20 +3408,20 @@ public void run() { } // get the links for a specific site - DigestURI url; + DigestURL url; try { - url = new DigestURI(r); + url = new DigestURL(r); } catch (final MalformedURLException e ) { ConcurrentLog.logException(e); return; } - final Map links; + final Map links; searchEvent.oneFeederStarted(); try { links = Switchboard.this.loader.loadLinks(url, CacheStrategy.NOCACHE, BlacklistType.SEARCH, ClientIdentification.yacyIntranetCrawlerAgent); if ( links != null ) { - final Iterator i = links.keySet().iterator(); + final Iterator i = links.keySet().iterator(); while ( i.hasNext() ) { if ( !i.next().getHost().endsWith(host) ) { i.remove(); @@ -3447,23 +3447,23 @@ public final void heuristicSearchResults(final String url) { public void run() { // get the links for a specific site - final DigestURI startUrl; + final DigestURL startUrl; try { - startUrl = new DigestURI(url); + startUrl = new DigestURL(url); } catch (final MalformedURLException e) { ConcurrentLog.logException(e); return; } - final Map links; - DigestURI url; + final Map links; + DigestURL url; try { links = Switchboard.this.loader.loadLinks(startUrl, CacheStrategy.IFFRESH, BlacklistType.SEARCH, ClientIdentification.yacyIntranetCrawlerAgent); if (links != null) { if (links.size() < 1000) { // limit to 1000 to skip large index pages - final Iterator i = links.keySet().iterator(); + final Iterator i = links.keySet().iterator(); final boolean globalcrawljob = Switchboard.this.getConfigBool(SwitchboardConstants.HEURISTIC_SEARCHRESULTS_CRAWLGLOBAL,false); - Collection urls = new ArrayList(); + Collection urls = new ArrayList(); while (i.hasNext()) { url = i.next(); boolean islocal = (url.getHost() == null && startUrl.getHost() == null) || (url.getHost() != null && startUrl.getHost() != null && url.getHost().contentEquals(startUrl.getHost())); @@ -3509,9 +3509,9 @@ public void run() { urlpattern.substring(0, p) + queryString.trim().replaceAll(" ", "+") + urlpattern.substring(p + 1); - final DigestURI url; + final DigestURL url; try { - url = new DigestURI(MultiProtocolURI.unescape(urlString)); + url = new DigestURL(MultiProtocolURL.unescape(urlString)); } catch (final MalformedURLException e1 ) { ConcurrentLog.warn("heuristicRSS", "url not well-formed: '" + urlString + "'"); return; @@ -3527,11 +3527,11 @@ public void run() { //System.out.println("BLEKKO: " + UTF8.String(resource)); rss = resource == null ? null : RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, resource); if ( rss != null ) { - final Map links = new TreeMap(); - DigestURI uri; + final Map links = new TreeMap(); + DigestURL uri; for ( final RSSMessage message : rss.getFeed() ) { try { - uri = new DigestURI(message.getLink()); + uri = new DigestURL(message.getLink()); links.put(uri, message.getTitle()); } catch (final MalformedURLException e ) { } @@ -3649,7 +3649,7 @@ private static void loadSeedListConcurrently(final SeedDB peers, final String se public void run() { // load the seed list try { - DigestURI url = new DigestURI(seedListFileURL); + DigestURL url = new DigestURL(seedListFileURL); //final long start = System.currentTimeMillis(); final RequestHeader reqHeader = new RequestHeader(); reqHeader.put(HeaderFramework.PRAGMA, "no-cache"); diff --git a/source/net/yacy/search/index/DocumentIndex.java b/source/net/yacy/search/index/DocumentIndex.java index 658e01fb14..cc4bed2079 100644 --- a/source/net/yacy/search/index/DocumentIndex.java +++ b/source/net/yacy/search/index/DocumentIndex.java @@ -34,13 +34,13 @@ import org.apache.solr.common.SolrInputDocument; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.Condenser; import net.yacy.document.Document; import net.yacy.document.LibraryProvider; import net.yacy.document.TextParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.workflow.WorkflowProcessor; import net.yacy.search.schema.CollectionConfiguration; import net.yacy.search.schema.WebgraphConfiguration; @@ -52,14 +52,14 @@ */ public class DocumentIndex extends Segment { - private static DigestURI poison; + private static DigestURL poison; static { try { - poison = new DigestURI("file://."); + poison = new DigestURL("file://."); } catch (final MalformedURLException e ) { } } - BlockingQueue queue; // a queue of document ID's + BlockingQueue queue; // a queue of document ID's private final Worker[] worker; CallbackListener callback; @@ -80,7 +80,7 @@ public DocumentIndex(final File segmentPath, final File collectionConfigurationP super.fulltext().connectLocalSolr(); super.fulltext().writeWebgraph(true); this.callback = callback; - this.queue = new LinkedBlockingQueue(WorkflowProcessor.availableCPU * 300); + this.queue = new LinkedBlockingQueue(WorkflowProcessor.availableCPU * 300); this.worker = new Worker[WorkflowProcessor.availableCPU]; for ( int i = 0; i < WorkflowProcessor.availableCPU; i++ ) { this.worker[i] = new Worker(i); @@ -96,7 +96,7 @@ public Worker(final int count) { @Override public void run() { - DigestURI f; + DigestURL f; SolrInputDocument[] resultRows; try { while ( (f = DocumentIndex.this.queue.take()) != poison ) { @@ -134,7 +134,7 @@ public void clearQueue() { this.queue.clear(); } - private SolrInputDocument[] add(final DigestURI url) throws IOException { + private SolrInputDocument[] add(final DigestURL url) throws IOException { if ( url == null ) { throw new IOException("file = null"); } @@ -183,7 +183,7 @@ private SolrInputDocument[] add(final DigestURI url) throws IOException { * * @param start */ - public void addConcurrent(final DigestURI start) throws IOException { + public void addConcurrent(final DigestURL start) throws IOException { assert (start != null); assert (start.canRead()) : start.toString(); if ( !start.isDirectory() ) { @@ -194,10 +194,10 @@ public void addConcurrent(final DigestURI start) throws IOException { return; } final String[] s = start.list(); - DigestURI w; + DigestURL w; for ( final String t : s ) { try { - w = new DigestURI(start, t); + w = new DigestURL(start, t); if ( w.canRead() && !w.isHidden() ) { if ( w.isDirectory() ) { addConcurrent(w); @@ -240,9 +240,9 @@ public synchronized void close() { public interface CallbackListener { - public void commit(DigestURI f, SolrInputDocument resultRow); + public void commit(DigestURL f, SolrInputDocument resultRow); - public void fail(DigestURI f, String failReason); + public void fail(DigestURL f, String failReason); } } diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index 423e2110c5..f3496e72f2 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -42,8 +42,9 @@ import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.solr.connector.AbstractSolrConnector; import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector; import net.yacy.cora.federate.solr.connector.RemoteSolrConnector; @@ -62,7 +63,6 @@ import net.yacy.cora.storage.ZIPWriter; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.parser.html.CharacterCoding; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.word.WordReferenceVars; @@ -319,7 +319,7 @@ public Date getLoadDate(final String urlHash) { return x; } - public DigestURI getURL(final byte[] urlHash) { + public DigestURL getURL(final byte[] urlHash) { if (urlHash == null || this.getDefaultConnector() == null) return null; String x; @@ -330,7 +330,7 @@ public DigestURI getURL(final byte[] urlHash) { } if (x == null) return null; try { - DigestURI uri = new DigestURI(x, urlHash); + DigestURL uri = new DigestURL(x, urlHash); return uri; } catch (final MalformedURLException e) { return null; @@ -531,8 +531,8 @@ public void deleteDomainHostname(final String hostname, Date freshdate) { * @param concurrently if true, then the method returnes immediately and runs concurrently */ public int remove(final String basepath, Date freshdate) { - DigestURI uri; - try {uri = new DigestURI(basepath);} catch (final MalformedURLException e) {return 0;} + DigestURL uri; + try {uri = new DigestURL(basepath);} catch (final MalformedURLException e) {return 0;} final String host = uri.getHost(); final String collectionQuery = CollectionSchema.host_s.getSolrFieldName() + ":\"" + host + "\"" + ((freshdate != null && freshdate.before(new Date())) ? (" AND " + CollectionSchema.load_date_dt.getSolrFieldName() + ":[* TO " + ISO8601Formatter.FORMATTER.format(freshdate) + "]") : ""); @@ -829,12 +829,12 @@ public void run() { pw.println(url); } if (this.format == 1) { - if (title != null) pw.println("" + CharacterCoding.unicode2xml((String) title.iterator().next(), true) + ""); + if (title != null) pw.println("" + CharacterCoding.unicode2xml((String) title.iterator().next(), true) + ""); } if (this.format == 2) { pw.println(""); if (title != null) pw.println("" + CharacterCoding.unicode2xml((String) title.iterator().next(), true) + ""); - pw.println("" + MultiProtocolURI.escape(url) + ""); + pw.println("" + MultiProtocolURL.escape(url) + ""); if (author != null && !author.isEmpty()) pw.println("" + CharacterCoding.unicode2xml(author, true) + ""); if (descriptions != null && descriptions.length > 0) { for (String d: descriptions) pw.println("" + CharacterCoding.unicode2xml(d, true) + ""); @@ -889,7 +889,7 @@ public Iterator statistics(int count, final ScoreMap domainSco count += 10; // make some more to prevent that we have to do this again after deletions too soon. if (count < 0 || domainScore.sizeSmaller(count)) count = domainScore.size(); this.statsDump = new ArrayList(); - DigestURI url; + DigestURL url; while (j.hasNext()) { urlhash = j.next(); if (urlhash == null) continue; diff --git a/source/net/yacy/search/index/Segment.java b/source/net/yacy/search/index/Segment.java index e6092b92e2..abf4fc67ea 100644 --- a/source/net/yacy/search/index/Segment.java +++ b/source/net/yacy/search/index/Segment.java @@ -41,9 +41,10 @@ import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrInputDocument; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.solr.connector.AbstractSolrConnector; import net.yacy.cora.federate.solr.connector.SolrConnector; import net.yacy.cora.federate.yacy.CacheStrategy; @@ -62,7 +63,6 @@ import net.yacy.document.Parser; import net.yacy.kelondro.data.citation.CitationReference; import net.yacy.kelondro.data.citation.CitationReferenceFactory; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.WordReference; @@ -216,10 +216,10 @@ public IndexCell urlCitation() { * @return the clickdepth level or 999 if the root url cannot be found or a recursion limit is reached * @throws IOException */ - public int getClickDepth(final DigestURI url) throws IOException { + public int getClickDepth(final DigestURL url) throws IOException { final byte[] searchhash = url.hash(); - RowHandleSet rootCandidates = url.getPossibleRootHashes(); + RowHandleSet rootCandidates = getPossibleRootHashes(url); RowHandleSet ignore = new RowHandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 100); // a set of urlhashes to be ignored. This is generated from all hashes that are seen during recursion to prevent enless loops RowHandleSet levelhashes = new RowHandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 1); // all hashes of a clickdepth. The first call contains the target hash only and therefore just one entry @@ -267,6 +267,25 @@ public int getClickDepth(final DigestURI url) throws IOException { } return 999; } + + private static RowHandleSet getPossibleRootHashes(DigestURL url) { + RowHandleSet rootCandidates = new RowHandleSet(URIMetadataRow.rowdef.primaryKeyLength, URIMetadataRow.rowdef.objectOrder, 10); + String rootStub = url.getProtocol() + "://" + url.getHost(); + try { + rootCandidates.put(new DigestURL(rootStub).hash()); + rootCandidates.put(new DigestURL(rootStub + "/").hash()); + rootCandidates.put(new DigestURL(rootStub + "/index.htm").hash()); + rootCandidates.put(new DigestURL(rootStub + "/index.html").hash()); + rootCandidates.put(new DigestURL(rootStub + "/index.php").hash()); + rootCandidates.put(new DigestURL(rootStub + "/home.htm").hash()); + rootCandidates.put(new DigestURL(rootStub + "/home.html").hash()); + rootCandidates.put(new DigestURL(rootStub + "/home.php").hash()); + rootCandidates.put(new DigestURL(rootStub + "/default.htm").hash()); + rootCandidates.put(new DigestURL(rootStub + "/default.html").hash()); + rootCandidates.put(new DigestURL(rootStub + "/default.php").hash()); + } catch (final Throwable e) {} + return rootCandidates; + } public ReferenceReportCache getReferenceReportCache() { return new ReferenceReportCache(); @@ -428,7 +447,7 @@ public Set exists(final Collection ids) { * @param stub * @return an iterator for all matching urls */ - public Iterator urlSelector(final MultiProtocolURI stub, final long maxtime, final int maxcount) { + public Iterator urlSelector(final MultiProtocolURL stub, final long maxtime, final int maxcount) { final BlockingQueue docQueue; final String urlstub; if (stub == null) { @@ -436,15 +455,15 @@ public Iterator urlSelector(final MultiProtocolURI stub, final long m urlstub = null; } else { final String host = stub.getHost(); - String hh = DigestURI.hosthash(host); + String hh = DigestURL.hosthash(host); docQueue = this.fulltext.getDefaultConnector().concurrentDocumentsByQuery(CollectionSchema.host_id_s + ":\"" + hh + "\"", 0, Integer.MAX_VALUE, maxtime, maxcount, CollectionSchema.id.getSolrFieldName(), CollectionSchema.sku.getSolrFieldName()); urlstub = stub.toNormalform(true); } // now filter the stub from the iterated urls - return new LookAheadIterator() { + return new LookAheadIterator() { @Override - protected DigestURI next0() { + protected DigestURL next0() { while (true) { SolrDocument doc; try { @@ -456,9 +475,9 @@ protected DigestURI next0() { if (doc == null || doc == AbstractSolrConnector.POISON_DOCUMENT) return null; String u = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()); String id = (String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()); - DigestURI url; + DigestURL url; try { - url = new DigestURI(u, ASCII.getBytes(id)); + url = new DigestURL(u, ASCII.getBytes(id)); } catch (final MalformedURLException e) { continue; } @@ -498,7 +517,7 @@ public synchronized void close() { } private static String votedLanguage( - final DigestURI url, + final DigestURL url, final String urlNormalform, final Document document, final Condenser condenser) { @@ -573,8 +592,8 @@ public void putDocumentInQueue(final SolrInputDocument queueEntry) { } public SolrInputDocument storeDocument( - final DigestURI url, - final DigestURI referrerURL, + final DigestURL url, + final DigestURL referrerURL, final Map collections, final ResponseHeader responseHeader, final Document document, @@ -635,7 +654,7 @@ public SolrInputDocument storeDocument( int outlinksSame = document.inboundLinks().size(); int outlinksOther = document.outboundLinks().size(); final int urlLength = urlNormalform.length(); - final int urlComps = MultiProtocolURI.urlComps(url.toString()).length; + final int urlComps = MultiProtocolURL.urlComps(url.toString()).length; // create a word prototype which is re-used for all entries if ((this.termIndex != null && storeToRWI) || searchEvent != null) { @@ -728,7 +747,7 @@ public int removeAllUrlReferences(final byte[] urlhash, final LoaderDispatcher l if (urlhash == null) return 0; // determine the url string - final DigestURI url = fulltext().getURL(urlhash); + final DigestURL url = fulltext().getURL(urlhash); if (url == null) return 0; try { diff --git a/source/net/yacy/search/query/AccessTracker.java b/source/net/yacy/search/query/AccessTracker.java index f299f2cecb..eee6e3d38b 100644 --- a/source/net/yacy/search/query/AccessTracker.java +++ b/source/net/yacy/search/query/AccessTracker.java @@ -35,8 +35,8 @@ import java.util.LinkedList; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.UTF8; import net.yacy.cora.document.WordCache; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.util.ConcurrentLog; import net.yacy.kelondro.util.MemoryControl; diff --git a/source/net/yacy/search/query/QueryModifier.java b/source/net/yacy/search/query/QueryModifier.java index e287c8f3f5..9908f86461 100644 --- a/source/net/yacy/search/query/QueryModifier.java +++ b/source/net/yacy/search/query/QueryModifier.java @@ -24,8 +24,8 @@ import org.apache.solr.common.params.CommonParams; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.util.CommonPattern; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.search.schema.CollectionSchema; import net.yacy.server.serverObjects; @@ -104,7 +104,7 @@ public String parse(String querystring) { while ( sitehost.endsWith(".") ) { sitehost = sitehost.substring(0, sitehost.length() - 1); } - sitehash = DigestURI.hosthash(sitehost); + sitehash = DigestURL.hosthash(sitehost); add("site:" + sitehost); } diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index 6fcdd6ef55..f4c5408396 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -40,9 +40,10 @@ import org.apache.solr.client.solrj.SolrQuery.SortClause; import org.apache.solr.common.params.FacetParams; -import net.yacy.cora.document.ASCII; import net.yacy.cora.document.analysis.Classification; import net.yacy.cora.document.analysis.Classification.ContentDomain; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.solr.Ranking; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.geo.GeoLocation; @@ -53,7 +54,6 @@ import net.yacy.cora.util.SpaceExceededException; import net.yacy.document.Condenser; import net.yacy.document.LibraryProvider; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.WordReferenceRow; import net.yacy.kelondro.index.RowHandleSet; @@ -148,7 +148,7 @@ public QueryParams( this.targetlang = "en"; this.metatags = new ArrayList(0); this.domType = Searchdom.LOCAL; - this.zonecode = DigestURI.TLD_any_zone_filter; + this.zonecode = DigestURL.TLD_any_zone_filter; this.constraint = constraint; this.allofconstraint = false; this.snippetCacheStrategy = null; @@ -550,11 +550,11 @@ public QueryGoal getQueryGoal() { return this.queryGoal; } - public final Map separateMatches(final Map links) { - final Map matcher = new HashMap(); - final Iterator > i = links.entrySet().iterator(); - Map.Entry entry; - DigestURI url; + public final Map separateMatches(final Map links) { + final Map matcher = new HashMap(); + final Iterator > i = links.entrySet().iterator(); + Map.Entry entry; + DigestURL url; String anchorText; while (i.hasNext()) { entry = i.next(); diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index 864318075b..a34300d826 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -44,11 +44,12 @@ import java.util.regex.Pattern; import net.yacy.contentcontrol.ContentControlFilterUpdateThread; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; import net.yacy.cora.document.analysis.Classification; import net.yacy.cora.document.analysis.Classification.ContentDomain; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.federate.yacy.Distribution; import net.yacy.cora.lod.vocabulary.Tagging; @@ -68,7 +69,6 @@ import net.yacy.document.LargeNumberCache; import net.yacy.document.LibraryProvider; import net.yacy.document.TextParser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.data.word.Word; @@ -96,6 +96,7 @@ import net.yacy.search.snippet.ResultEntry; import net.yacy.search.snippet.TextSnippet; import net.yacy.search.snippet.TextSnippet.ResultClass; + import org.apache.solr.common.SolrDocument; public final class SearchEvent { @@ -844,7 +845,7 @@ public void addNodes( } // filter out media links in text search, if wanted - String ext = MultiProtocolURI.getFileExtension(iEntry.url().getFileName()); + String ext = MultiProtocolURL.getFileExtension(iEntry.url().getFileName()); if (this.query.contentdom == ContentDomain.TEXT && Classification.isImageExtension(ext) && this.excludeintext_image) { if (log.isFine()) log.fine("dropped Node: file name domain does not match"); continue pollloop; @@ -1037,7 +1038,7 @@ public URIMetadataNode pullOneFilteredFromRWI(final boolean skipDoubleDom) { } // filter out media links in text search, if wanted - String ext = MultiProtocolURI.getFileExtension(page.url().getFileName()); + String ext = MultiProtocolURL.getFileExtension(page.url().getFileName()); if (this.query.contentdom == ContentDomain.TEXT && Classification.isImageExtension(ext) && this.excludeintext_image) { if (log.isFine()) log.fine("dropped RWI: file name domain does not match"); continue; @@ -1267,8 +1268,8 @@ private long postRanking(final ResultEntry rentry, final ScoreMap topwor // apply 'common-sense' heuristic using references final String urlstring = rentry.url().toNormalform(true); - final String[] urlcomps = MultiProtocolURI.urlComps(urlstring); - final String[] descrcomps = MultiProtocolURI.splitpattern.split(rentry.title().toLowerCase()); + final String[] urlcomps = MultiProtocolURL.urlComps(urlstring); + final String[] descrcomps = MultiProtocolURL.splitpattern.split(rentry.title().toLowerCase()); for (final String urlcomp : urlcomps) { int tc = topwords.get(urlcomp); if (tc > 0) r += Math.max(1, tc) << this.query.ranking.coeff_urlcompintoplist; @@ -1422,7 +1423,7 @@ public ImageResult oneImageResult(final int item, final long timeout) throws Mal String a = alt != null && alt.size() > c ? (String) SetTools.nth(alt, c) : ""; if (query.getQueryGoal().matches((String) i) || query.getQueryGoal().matches(a)) { try { - DigestURI imageUrl = new DigestURI((prt != null && prt.size() > c ? SetTools.nth(prt, c) : "http") + "://" + i); + DigestURL imageUrl = new DigestURL((prt != null && prt.size() > c ? SetTools.nth(prt, c) : "http") + "://" + i); Object heightO = SetTools.nth(doc.getFieldValues(CollectionSchema.images_height_val.getSolrFieldName()), c); Object widthO = SetTools.nth(doc.getFieldValues(CollectionSchema.images_width_val.getSolrFieldName()), c); String id = ASCII.String(imageUrl.hash()); @@ -1434,12 +1435,12 @@ public ImageResult oneImageResult(final int item, final long timeout) throws Mal c++; } } - if (MultiProtocolURI.isImage(MultiProtocolURI.getFileExtension(ms.url().getFileName()))) { + if (MultiProtocolURL.isImage(MultiProtocolURL.getFileExtension(ms.url().getFileName()))) { String id = ASCII.String(ms.hash()); if (!imageViewed.containsKey(id) && !imageSpare.containsKey(id)) imageSpare.put(id, new ImageResult(ms.url(), ms.url(), "", ms.title(), 0, 0, 0)); } if (img != null && img.size() > 0) { - DigestURI imageUrl = new DigestURI((prt != null && prt.size() > 0 ? SetTools.nth(prt, 0) : "http") + "://" + SetTools.nth(img, 0)); + DigestURL imageUrl = new DigestURL((prt != null && prt.size() > 0 ? SetTools.nth(prt, 0) : "http") + "://" + SetTools.nth(img, 0)); String imagetext = alt != null && alt.size() > 0 ? (String) SetTools.nth(alt, 0) : ""; String id = ASCII.String(imageUrl.hash()); if (!imageViewed.containsKey(id) && !imageSpare.containsKey(id)) imageSpare.put(id, new ImageResult(ms.url(), imageUrl, "", imagetext, 0, 0, 0)); @@ -1450,10 +1451,10 @@ public ImageResult oneImageResult(final int item, final long timeout) throws Mal } public class ImageResult { - public DigestURI imageUrl, sourceUrl; + public DigestURL imageUrl, sourceUrl; public String mimetype = "", imagetext = ""; public int width = 0, height = 0, fileSize = 0; - public ImageResult(DigestURI sourceUrl, DigestURI imageUrl, String mimetype, String imagetext, int width, int height, int fileSize) { + public ImageResult(DigestURL sourceUrl, DigestURL imageUrl, String mimetype, String imagetext, int width, int height, int fileSize) { this.sourceUrl = sourceUrl; this.imageUrl = imageUrl; this.mimetype = mimetype; @@ -1638,7 +1639,7 @@ public void addTopic(final String[] words) { protected void addTopics(final ResultEntry resultEntry) { // take out relevant information for reference computation if ((resultEntry.url() == null) || (resultEntry.title() == null)) return; - final String[] descrcomps = MultiProtocolURI.splitpattern.split(resultEntry.title().toLowerCase()); // words in the description + final String[] descrcomps = MultiProtocolURL.splitpattern.split(resultEntry.title().toLowerCase()); // words in the description // add references addTopic(descrcomps); diff --git a/source/net/yacy/search/ranking/BlockRank.java b/source/net/yacy/search/ranking/BlockRank.java index 3dd0f407ed..c59e5d743f 100644 --- a/source/net/yacy/search/ranking/BlockRank.java +++ b/source/net/yacy/search/ranking/BlockRank.java @@ -32,7 +32,7 @@ import java.util.List; import java.util.Map; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.order.Base64Order; import net.yacy.cora.sorting.OrderedScoreMap; import net.yacy.cora.sorting.ScoreMap; diff --git a/source/net/yacy/search/ranking/ReferenceOrder.java b/source/net/yacy/search/ranking/ReferenceOrder.java index d3c27e70a0..10fd7373cf 100644 --- a/source/net/yacy/search/ranking/ReferenceOrder.java +++ b/source/net/yacy/search/ranking/ReferenceOrder.java @@ -33,12 +33,12 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Semaphore; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.sorting.ConcurrentScoreMap; import net.yacy.cora.util.ByteBuffer; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.Condenser; import net.yacy.document.LargeNumberCache; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.data.word.WordReferenceRow; @@ -229,7 +229,7 @@ public long cardinal(final WordReference t) { final int maxmaxpos = this.max.maxposition(); final int minminpos = this.min.minposition(); final long r = - ((256 - DigestURI.domLengthNormalized(t.urlhash())) << this.ranking.coeff_domlength) + ((256 - DigestURL.domLengthNormalized(t.urlhash())) << this.ranking.coeff_domlength) + ((this.max.urlcomps() == this.min.urlcomps() ) ? 0 : (256 - (((t.urlcomps() - this.min.urlcomps() ) << 8) / (this.max.urlcomps() - this.min.urlcomps()) )) << this.ranking.coeff_urlcomps) + ((this.max.urllength() == this.min.urllength() ) ? 0 : (256 - (((t.urllength() - this.min.urllength() ) << 8) / (this.max.urllength() - this.min.urllength()) )) << this.ranking.coeff_urllength) + ((maxmaxpos == minminpos) ? 0 : (256 - (((t.minposition() - minminpos) << 8) / (maxmaxpos - minminpos))) << this.ranking.coeff_posintext) @@ -270,7 +270,7 @@ public long cardinal(final URIMetadataNode t) { assert t != null; assert this.ranking != null; final long r = - ((256 - DigestURI.domLengthNormalized(t.hash())) << this.ranking.coeff_domlength) + ((256 - DigestURL.domLengthNormalized(t.hash())) << this.ranking.coeff_domlength) + ((256 - (t.urllength() << 8)) << this.ranking.coeff_urllength) + (t.virtualAge() << this.ranking.coeff_date) + (t.wordsintitle()<< this.ranking.coeff_wordsintitle) diff --git a/source/net/yacy/search/schema/CollectionConfiguration.java b/source/net/yacy/search/schema/CollectionConfiguration.java index 107c04686f..9c2db013d2 100644 --- a/source/net/yacy/search/schema/CollectionConfiguration.java +++ b/source/net/yacy/search/schema/CollectionConfiguration.java @@ -35,19 +35,20 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.BlockingQueue; import java.util.regex.Pattern; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; import net.yacy.cora.document.analysis.EnhancedTextProfileSignature; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.solr.Ranking; import net.yacy.cora.federate.solr.SchemaConfiguration; import net.yacy.cora.federate.solr.FailType; @@ -72,7 +73,6 @@ import net.yacy.document.parser.html.ContentScraper; import net.yacy.document.parser.html.ImageEntry; import net.yacy.kelondro.data.citation.CitationReference; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataRow; import net.yacy.kelondro.index.RowHandleMap; import net.yacy.kelondro.rwi.IndexCell; @@ -204,7 +204,7 @@ public SolrDocument toSolrDocument(final SolrInputDocument doc) { * @param doctype * @return the normalized url */ - public String addURIAttributes(final SolrInputDocument doc, final boolean allAttr, final DigestURI digestURI, final char doctype) { + public String addURIAttributes(final SolrInputDocument doc, final boolean allAttr, final DigestURL digestURI, final char doctype) { add(doc, CollectionSchema.id, ASCII.String(digestURI.hash())); String us = digestURI.toNormalform(true); add(doc, CollectionSchema.sku, us); @@ -228,7 +228,7 @@ public String addURIAttributes(final SolrInputDocument doc, final boolean allAtt // path elements of link String filename = digestURI.getFileName(); - String extension = MultiProtocolURI.getFileExtension(filename); + String extension = MultiProtocolURL.getFileExtension(filename); if (allAttr || contains(CollectionSchema.url_chars_i)) add(doc, CollectionSchema.url_chars_i, us.length()); if (allAttr || contains(CollectionSchema.url_protocol_s)) add(doc, CollectionSchema.url_protocol_s, digestURI.getProtocol()); if (allAttr || contains(CollectionSchema.url_paths_sxt)) add(doc, CollectionSchema.url_paths_sxt, digestURI.getPaths()); @@ -357,12 +357,12 @@ public List getWebgraphDocuments() { public SolrVector yacy2solr( final String id, final Map collections, final ResponseHeader responseHeader, - final Document document, final Condenser condenser, final DigestURI referrerURL, final String language, + final Document document, final Condenser condenser, final DigestURL referrerURL, final String language, final IndexCell citations, final WebgraphConfiguration webgraph) { // we use the SolrCell design as index schema SolrVector doc = new SolrVector(); - final DigestURI digestURI = document.dc_source(); + final DigestURL digestURI = document.dc_source(); boolean allAttr = this.isEmpty(); String url = addURIAttributes(doc, allAttr, digestURI, Response.docType(digestURI)); @@ -464,12 +464,11 @@ public SolrVector yacy2solr( } // get list of all links; they will be shrinked by urls that appear in other fields of the solr schema - Set inboundLinks = document.inboundLinks(); - Set outboundLinks = document.outboundLinks(); + LinkedHashMap inboundLinks = document.inboundLinks(); + LinkedHashMap outboundLinks = document.outboundLinks(); Subgraph subgraph = new Subgraph(inboundLinks.size(), outboundLinks.size()); - Map alllinks = document.getAnchors(); - Map images = new HashMap(); + List images = new ArrayList(); int c = 0; final Object parser = document.getParserObject(); boolean containsCanonical = false; @@ -582,18 +581,17 @@ public SolrVector yacy2solr( if (li.length > 0) add(doc, CollectionSchema.li_txt, li); // images - final Collection imagesc = images.values(); - final ArrayList imgprots = new ArrayList(imagesc.size()); - final Integer[] imgheights = new Integer[imagesc.size()]; - final Integer[] imgwidths = new Integer[imagesc.size()]; - final Integer[] imgpixels = new Integer[imagesc.size()]; - final String[] imgstubs = new String[imagesc.size()]; - final String[] imgalts = new String[imagesc.size()]; + final ArrayList imgprots = new ArrayList(images.size()); + final Integer[] imgheights = new Integer[images.size()]; + final Integer[] imgwidths = new Integer[images.size()]; + final Integer[] imgpixels = new Integer[images.size()]; + final String[] imgstubs = new String[images.size()]; + final String[] imgalts = new String[images.size()]; int withalt = 0; int i = 0; LinkedHashSet images_text_map = new LinkedHashSet(); - for (final ImageEntry ie: imagesc) { - final MultiProtocolURI uri = ie.url(); + for (final ImageEntry ie: images) { + final MultiProtocolURL uri = ie.url(); inboundLinks.remove(uri); outboundLinks.remove(uri); imgheights[i] = ie.height(); @@ -613,7 +611,7 @@ public SolrVector yacy2solr( } StringBuilder images_text = new StringBuilder(images_text_map.size() * 6 + 1); for (String s: images_text_map) images_text.append(s.trim()).append(' '); - if (allAttr || contains(CollectionSchema.imagescount_i)) add(doc, CollectionSchema.imagescount_i, imagesc.size()); + if (allAttr || contains(CollectionSchema.imagescount_i)) add(doc, CollectionSchema.imagescount_i, images.size()); if (allAttr || contains(CollectionSchema.images_protocol_sxt)) add(doc, CollectionSchema.images_protocol_sxt, protocolList2indexedList(imgprots)); if (allAttr || contains(CollectionSchema.images_urlstub_sxt)) add(doc, CollectionSchema.images_urlstub_sxt, imgstubs); if (allAttr || contains(CollectionSchema.images_alt_sxt)) add(doc, CollectionSchema.images_alt_sxt, imgalts); @@ -625,11 +623,11 @@ public SolrVector yacy2solr( // style sheets if (allAttr || contains(CollectionSchema.css_tag_sxt)) { - final Map csss = html.getCSS(); + final Map csss = html.getCSS(); final String[] css_tag = new String[csss.size()]; final String[] css_url = new String[csss.size()]; c = 0; - for (final Map.Entry entry: csss.entrySet()) { + for (final Map.Entry entry: csss.entrySet()) { final String cssurl = entry.getKey().toNormalform(false); inboundLinks.remove(cssurl); outboundLinks.remove(cssurl); @@ -646,10 +644,10 @@ public SolrVector yacy2solr( // Scripts if (allAttr || contains(CollectionSchema.scripts_sxt)) { - final Set scriptss = html.getScript(); + final Set scriptss = html.getScript(); final String[] scripts = new String[scriptss.size()]; c = 0; - for (final DigestURI u: scriptss) { + for (final DigestURL u: scriptss) { inboundLinks.remove(u); outboundLinks.remove(u); scripts[c++] = u.toNormalform(false); @@ -660,10 +658,10 @@ public SolrVector yacy2solr( // Frames if (allAttr || contains(CollectionSchema.frames_sxt)) { - final Set framess = html.getFrames(); + final Set framess = html.getFrames(); final String[] frames = new String[framess.size()]; c = 0; - for (final DigestURI u: framess) { + for (final DigestURL u: framess) { inboundLinks.remove(u); outboundLinks.remove(u); frames[c++] = u.toNormalform(false); @@ -677,10 +675,10 @@ public SolrVector yacy2solr( // IFrames if (allAttr || contains(CollectionSchema.iframes_sxt)) { - final Set iframess = html.getIFrames(); + final Set iframess = html.getIFrames(); final String[] iframes = new String[iframess.size()]; c = 0; - for (final DigestURI u: iframess) { + for (final DigestURL u: iframess) { inboundLinks.remove(u); outboundLinks.remove(u); iframes[c++] = u.toNormalform(false); @@ -694,7 +692,7 @@ public SolrVector yacy2solr( // canonical tag if (allAttr || contains(CollectionSchema.canonical_s)) { - final DigestURI canonical = html.getCanonical(); + final DigestURL canonical = html.getCanonical(); if (canonical != null && !ASCII.String(canonical.hash()).equals(id)) { containsCanonical = true; inboundLinks.remove(canonical); @@ -711,9 +709,9 @@ public SolrVector yacy2solr( if (allAttr || contains(CollectionSchema.refresh_s)) { String refresh = html.getRefreshPath(); if (refresh != null && refresh.length() > 0) { - MultiProtocolURI refreshURL; + MultiProtocolURL refreshURL; try { - refreshURL = refresh.startsWith("http") ? new MultiProtocolURI(html.getRefreshPath()) : new MultiProtocolURI(digestURI, html.getRefreshPath()); + refreshURL = refresh.startsWith("http") ? new MultiProtocolURL(html.getRefreshPath()) : new MultiProtocolURL(digestURI, html.getRefreshPath()); if (refreshURL != null) { inboundLinks.remove(refreshURL); outboundLinks.remove(refreshURL); @@ -727,8 +725,8 @@ public SolrVector yacy2solr( // flash embedded if (allAttr || contains(CollectionSchema.flash_b)) { - MultiProtocolURI[] flashURLs = html.getFlash(); - for (MultiProtocolURI u: flashURLs) { + MultiProtocolURL[] flashURLs = html.getFlash(); + for (MultiProtocolURL u: flashURLs) { // remove all flash links from ibound/outbound links inboundLinks.remove(u); outboundLinks.remove(u); @@ -755,7 +753,7 @@ public SolrVector yacy2solr( final String[] ccs = new String[html.getHreflang().size()]; final String[] urls = new String[html.getHreflang().size()]; c = 0; - for (Map.Entry e: html.getHreflang().entrySet()) { + for (Map.Entry e: html.getHreflang().entrySet()) { ccs[c] = e.getKey(); urls[c] = e.getValue().toNormalform(true); c++; @@ -769,7 +767,7 @@ public SolrVector yacy2solr( final String[] navs = new String[html.getNavigation().size()]; final String[] urls = new String[html.getNavigation().size()]; c = 0; - for (Map.Entry e: html.getNavigation().entrySet()) { + for (Map.Entry e: html.getNavigation().entrySet()) { navs[c] = e.getKey(); urls[c] = e.getValue().toNormalform(true); c++; @@ -790,7 +788,7 @@ public SolrVector yacy2solr( content = digestURI.toTokens(); } - if ((allAttr || contains(CollectionSchema.images_text_t)) && MultiProtocolURI.isImage(MultiProtocolURI.getFileExtension(digestURI.getFileName()))) { + if ((allAttr || contains(CollectionSchema.images_text_t)) && MultiProtocolURL.isImage(MultiProtocolURL.getFileExtension(digestURI.getFileName()))) { add(doc, CollectionSchema.images_text_t, content); // the content may contain the exif data from the image parser content = digestURI.toTokens(); // remove all other entry but the url tokens } @@ -816,8 +814,7 @@ public SolrVector yacy2solr( // create a subgraph if (!containsCanonical) { // a document with canonical tag should not get a webgraph relation, because that belongs to the canonical document - webgraph.addEdges(subgraph, digestURI, responseHeader, collections, clickdepth, alllinks, images, true, inboundLinks, citations); - webgraph.addEdges(subgraph, digestURI, responseHeader, collections, clickdepth, alllinks, images, false, outboundLinks, citations); + webgraph.addEdges(subgraph, digestURI, responseHeader, collections, clickdepth, images, true, document.getAnchors(), citations); } // list all links @@ -919,7 +916,7 @@ public int postprocessing(final Segment segment) { Collection proctags = doc.getFieldValues(CollectionSchema.process_sxt.getSolrFieldName()); try { - DigestURI url = new DigestURI((String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()), ASCII.getBytes((String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()))); + DigestURL url = new DigestURL((String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()), ASCII.getBytes((String) doc.getFieldValue(CollectionSchema.id.getSolrFieldName()))); byte[] id = url.hash(); SolrInputDocument sid = this.toSolrInputDocument(doc); @@ -1206,7 +1203,7 @@ private static List relEval(final List rel) { * @param httpstatus * @throws IOException */ - public SolrInputDocument err(final DigestURI digestURI, final Map collections, final String failReason, final FailType failType, final int httpstatus) throws IOException { + public SolrInputDocument err(final DigestURL digestURI, final Map collections, final String failReason, final FailType failType, final int httpstatus) throws IOException { boolean allAttr = this.isEmpty(); assert allAttr || contains(CollectionSchema.failreason_s); diff --git a/source/net/yacy/search/schema/WebgraphConfiguration.java b/source/net/yacy/search/schema/WebgraphConfiguration.java index e3bf994efc..ad9a1c2414 100644 --- a/source/net/yacy/search/schema/WebgraphConfiguration.java +++ b/source/net/yacy/search/schema/WebgraphConfiguration.java @@ -42,8 +42,10 @@ import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrInputDocument; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.AnchorURL; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.solr.ProcessType; import net.yacy.cora.federate.solr.SchemaConfiguration; import net.yacy.cora.federate.solr.SchemaDeclaration; @@ -55,7 +57,6 @@ import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.parser.html.ImageEntry; import net.yacy.kelondro.data.citation.CitationReference; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.rwi.IndexCell; import net.yacy.search.index.Segment; @@ -115,16 +116,15 @@ public Subgraph(int inboundSize, int outboundSize) { public void addEdges( final Subgraph subgraph, - final DigestURI source, final ResponseHeader responseHeader, Map collections, int clickdepth_source, - final Map alllinks, final Map images, - final boolean inbound, final Set links, + final DigestURL source, final ResponseHeader responseHeader, Map collections, int clickdepth_source, + final List images, final boolean inbound, final Collection links, final IndexCell citations) { boolean allAttr = this.isEmpty(); - for (final DigestURI target_url: links) { + for (final AnchorURL target_url: links) { Set processTypes = new LinkedHashSet(); - final Properties p = alllinks.get(target_url); + final Properties p = target_url.getProperties(); if (p == null) continue; final String name = p.getProperty("name", ""); // the name attribute final String text = p.getProperty("text", ""); // the text between the tag @@ -186,7 +186,7 @@ public void addEdges( } if (allAttr || contains(WebgraphSchema.source_file_ext_s) || contains(WebgraphSchema.source_file_name_s)) { String source_file_name = source.getFileName(); - String source_file_ext = MultiProtocolURI.getFileExtension(source_file_name); + String source_file_ext = MultiProtocolURL.getFileExtension(source_file_name); add(edge, WebgraphSchema.source_file_name_s, source_file_name.toLowerCase().endsWith("." + source_file_ext) ? source_file_name.substring(0, source_file_name.length() - source_file_ext.length() - 1) : source_file_name); add(edge, WebgraphSchema.source_file_ext_s, source_file_ext); } @@ -209,7 +209,11 @@ public void addEdges( if (allAttr || contains(WebgraphSchema.target_linktext_t)) add(edge, WebgraphSchema.target_linktext_t, text.length() > 0 ? text : ""); if (allAttr || contains(WebgraphSchema.target_linktext_charcount_i)) add(edge, WebgraphSchema.target_linktext_charcount_i, text.length()); if (allAttr || contains(WebgraphSchema.target_linktext_wordcount_i)) add(edge, WebgraphSchema.target_linktext_wordcount_i, text.length() > 0 ? CommonPattern.SPACE.split(text).length : 0); - ImageEntry ientry = images.get(target_url); + + ImageEntry ientry = null; + for (ImageEntry ie: images) { + if (ie.linkurl() != null && ie.linkurl().equals(target_url)) {ientry = ie; break;} + } String alttext = ientry == null ? "" : ientry.alt(); if (allAttr || contains(WebgraphSchema.target_alt_t)) add(edge, WebgraphSchema.target_alt_t, alttext); if (allAttr || contains(WebgraphSchema.target_alt_charcount_i)) add(edge, WebgraphSchema.target_alt_charcount_i, alttext.length()); @@ -248,7 +252,7 @@ public void addEdges( } if (allAttr || contains(WebgraphSchema.target_file_ext_s) || contains(WebgraphSchema.target_file_name_s)) { String target_file_name = target_url.getFileName(); - String target_file_ext = MultiProtocolURI.getFileExtension(target_file_name); + String target_file_ext = MultiProtocolURL.getFileExtension(target_file_name); add(edge, WebgraphSchema.target_file_name_s, target_file_name.toLowerCase().endsWith("." + target_file_ext) ? target_file_name.substring(0, target_file_name.length() - target_file_ext.length() - 1) : target_file_name); add(edge, WebgraphSchema.target_file_ext_s, target_file_ext); } @@ -295,7 +299,7 @@ public int postprocessing(Segment segment) { SolrDocument doc; String protocol, urlstub, id; - DigestURI url; + DigestURL url; int proccount = 0, proccount_clickdepthchange = 0; try { while ((doc = docs.take()) != AbstractSolrConnector.POISON_DOCUMENT) { @@ -313,14 +317,14 @@ public int postprocessing(Segment segment) { protocol = (String) doc.getFieldValue(WebgraphSchema.source_protocol_s.getSolrFieldName()); urlstub = (String) doc.getFieldValue(WebgraphSchema.source_urlstub_s.getSolrFieldName()); id = (String) doc.getFieldValue(WebgraphSchema.source_id_s.getSolrFieldName()); - url = new DigestURI(protocol + "://" + urlstub, ASCII.getBytes(id)); + url = new DigestURL(protocol + "://" + urlstub, ASCII.getBytes(id)); if (postprocessing_clickdepth(segment, doc, sid, url, WebgraphSchema.source_clickdepth_i)) proccount_clickdepthchange++; } if (this.contains(WebgraphSchema.target_protocol_s) && this.contains(WebgraphSchema.target_urlstub_s) && this.contains(WebgraphSchema.target_id_s)) { protocol = (String) doc.getFieldValue(WebgraphSchema.target_protocol_s.getSolrFieldName()); urlstub = (String) doc.getFieldValue(WebgraphSchema.target_urlstub_s.getSolrFieldName()); id = (String) doc.getFieldValue(WebgraphSchema.target_id_s.getSolrFieldName()); - url = new DigestURI(protocol + "://" + urlstub, ASCII.getBytes(id)); + url = new DigestURL(protocol + "://" + urlstub, ASCII.getBytes(id)); if (postprocessing_clickdepth(segment, doc, sid, url, WebgraphSchema.target_clickdepth_i)) proccount_clickdepthchange++; } } diff --git a/source/net/yacy/search/snippet/MediaSnippet.java b/source/net/yacy/search/snippet/MediaSnippet.java index 988c37e6a6..0ac454cde4 100644 --- a/source/net/yacy/search/snippet/MediaSnippet.java +++ b/source/net/yacy/search/snippet/MediaSnippet.java @@ -35,9 +35,10 @@ import java.util.SortedSet; import java.util.TreeSet; -import net.yacy.cora.document.ASCII; import net.yacy.cora.document.analysis.Classification; import net.yacy.cora.document.analysis.Classification.ContentDomain; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.ClientIdentification; @@ -52,7 +53,6 @@ import net.yacy.document.Parser; import net.yacy.document.WordTokenizer; import net.yacy.document.parser.html.ImageEntry; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.index.RowHandleSet; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; @@ -60,13 +60,13 @@ public class MediaSnippet implements Comparable, Comparator { public ContentDomain type; - public DigestURI href, source; + public DigestURL href, source; public String name, attr, mime; public long ranking; public int width, height; public long fileSize; - public MediaSnippet(final ContentDomain type, final DigestURI href, final String mime, final String name, final long fileSize, final String attr, final long ranking, final DigestURI source) { + public MediaSnippet(final ContentDomain type, final DigestURL href, final String mime, final String name, final long fileSize, final String attr, final long ranking, final DigestURL source) { this.type = type; this.href = href; this.mime = mime; @@ -86,7 +86,7 @@ public MediaSnippet(final ContentDomain type, final DigestURI href, final String if ((this.attr == null) || (this.attr.isEmpty())) this.attr = "_"; } - public MediaSnippet(final ContentDomain type, final DigestURI href, final String mime, final String name, final long fileSize, final int width, final int height, final long ranking, final DigestURI source) { + public MediaSnippet(final ContentDomain type, final DigestURL href, final String mime, final String name, final long fileSize, final int width, final int height, final long ranking, final DigestURL source) { this.type = type; this.href = href; this.mime = mime; @@ -135,7 +135,7 @@ public int compare(final MediaSnippet o1, final MediaSnippet o2) { return o1.compareTo(o2); } - public static List retrieveMediaSnippets(final DigestURI url, final HandleSet queryhashes, final Classification.ContentDomain mediatype, final CacheStrategy cacheStrategy, final boolean reindexing) { + public static List retrieveMediaSnippets(final DigestURL url, final HandleSet queryhashes, final Classification.ContentDomain mediatype, final CacheStrategy cacheStrategy, final boolean reindexing) { if (queryhashes.isEmpty()) { ConcurrentLog.fine("snippet fetch", "no query hashes given for url " + url); return new ArrayList(); @@ -161,18 +161,18 @@ public static List retrieveMediaSnippets(final DigestURI url, fina return a; } - public static List computeMediaSnippets(final DigestURI source, final Document document, final HandleSet queryhashes, final ContentDomain mediatype) { + public static List computeMediaSnippets(final DigestURL source, final Document document, final HandleSet queryhashes, final ContentDomain mediatype) { if (document == null) return new ArrayList(); - Map media = null; + Map media = null; if (mediatype == ContentDomain.AUDIO) media = document.getAudiolinks(); else if (mediatype == ContentDomain.VIDEO) media = document.getVideolinks(); else if (mediatype == ContentDomain.APP) media = document.getApplinks(); if (media == null) return null; - final Iterator> i = media.entrySet().iterator(); - Map.Entry entry; - DigestURI url; + final Iterator> i = media.entrySet().iterator(); + Map.Entry entry; + DigestURL url; String desc; final List result = new ArrayList(); while (i.hasNext()) { @@ -189,7 +189,7 @@ public static List computeMediaSnippets(final DigestURI source, fi return result; } - public static List computeImageSnippets(final DigestURI source, final Document document, final HandleSet queryhashes) { + public static List computeImageSnippets(final DigestURL source, final Document document, final HandleSet queryhashes) { final SortedSet images = new TreeSet(); images.addAll(document.getImages().values()); // iterates images in descending size order! @@ -197,7 +197,7 @@ public static List computeImageSnippets(final DigestURI source, fi final Iterator i = images.iterator(); ImageEntry ientry; - DigestURI url; + DigestURL url; String desc; final List result = new ArrayList(); while (i.hasNext()) { @@ -253,7 +253,7 @@ private static HandleSet removeAppearanceHashes(final String sentence, final Han * @param blacklistType Type of blacklist (see class Blacklist, BLACKLIST_FOO) * @return isBlacklisted Wether the given URL is blacklisted */ - private static boolean isUrlBlacklisted (final BlacklistType blacklistType, final DigestURI url) { + private static boolean isUrlBlacklisted (final BlacklistType blacklistType, final DigestURL url) { // Default is not blacklisted boolean isBlacklisted = false; diff --git a/source/net/yacy/search/snippet/ResultEntry.java b/source/net/yacy/search/snippet/ResultEntry.java index fe8099416c..de5e4d95f7 100644 --- a/source/net/yacy/search/snippet/ResultEntry.java +++ b/source/net/yacy/search/snippet/ResultEntry.java @@ -31,12 +31,12 @@ import java.util.Date; import java.util.List; -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.order.Base64Order; import net.yacy.cora.util.ByteArray; import net.yacy.cora.util.ConcurrentLog; import net.yacy.document.Condenser; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.WordReference; @@ -128,7 +128,7 @@ public URIMetadataNode getNode() { public byte[] hash() { return this.urlentry.hash(); } - public DigestURI url() { + public DigestURL url() { return this.urlentry.url(); } public Bitfield flags() { @@ -138,7 +138,7 @@ public String urlstring() { return (this.alternative_urlstring == null) ? this.urlentry.url().toNormalform(true) : this.alternative_urlstring; } public String urlname() { - return (this.alternative_urlname == null) ? MultiProtocolURI.unescape(this.urlentry.url().toNormalform(true)) : this.alternative_urlname; + return (this.alternative_urlname == null) ? MultiProtocolURL.unescape(this.urlentry.url().toNormalform(true)) : this.alternative_urlname; } public String title() { return this.urlentry.dc_title(); diff --git a/source/net/yacy/search/snippet/TextSnippet.java b/source/net/yacy/search/snippet/TextSnippet.java index dfeaf20e2b..85e0bbc4f4 100644 --- a/source/net/yacy/search/snippet/TextSnippet.java +++ b/source/net/yacy/search/snippet/TextSnippet.java @@ -33,7 +33,8 @@ import java.util.Set; import java.util.regex.Pattern; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.ClientIdentification; @@ -50,7 +51,6 @@ import net.yacy.document.SnippetExtractor; import net.yacy.document.WordTokenizer; import net.yacy.document.parser.html.CharacterCoding; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.kelondro.data.word.Word; import net.yacy.peers.RemoteSearch; @@ -156,7 +156,7 @@ public TextSnippet( final boolean reindexing) { // heise = "0OQUNU3JSs05" - final DigestURI url = row.url(); + final DigestURL url = row.url(); if (queryhashes.isEmpty()) { //System.out.println("found no queryhashes for URL retrieve " + url); init(url.hash(), null, false, ResultClass.ERROR_NO_HASH_GIVEN, "no query hashes given"); diff --git a/source/net/yacy/server/http/AugmentedHtmlStream.java b/source/net/yacy/server/http/AugmentedHtmlStream.java index 66d32d9a32..1f43080931 100644 --- a/source/net/yacy/server/http/AugmentedHtmlStream.java +++ b/source/net/yacy/server/http/AugmentedHtmlStream.java @@ -9,19 +9,19 @@ import java.io.Writer; import java.nio.charset.Charset; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.RequestHeader; import net.yacy.interaction.AugmentHtmlStream; -import net.yacy.kelondro.data.meta.DigestURI; public class AugmentedHtmlStream extends FilterOutputStream { private final Writer out; private final ByteArrayOutputStream buffer; private final Charset charset; - private final DigestURI url; + private final DigestURL url; private final String urls; private final RequestHeader requestHeader; - public AugmentedHtmlStream(OutputStream out, Charset charset, DigestURI url, RequestHeader requestHeader) { + public AugmentedHtmlStream(OutputStream out, Charset charset, DigestURL url, RequestHeader requestHeader) { super(out); this.out = new BufferedWriter(new OutputStreamWriter(out, charset)); this.buffer = new ByteArrayOutputStream(); diff --git a/source/net/yacy/server/http/ChunkedOutputStream.java b/source/net/yacy/server/http/ChunkedOutputStream.java index d928caed73..108416d7f5 100644 --- a/source/net/yacy/server/http/ChunkedOutputStream.java +++ b/source/net/yacy/server/http/ChunkedOutputStream.java @@ -29,8 +29,8 @@ import java.io.InputStream; import java.io.OutputStream; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.util.ByteBuffer; import net.yacy.kelondro.util.FileUtils; import net.yacy.server.serverCore; diff --git a/source/net/yacy/server/http/HTTPDFileHandler.java b/source/net/yacy/server/http/HTTPDFileHandler.java index 0a834905ee..e6c7dcaf3a 100644 --- a/source/net/yacy/server/http/HTTPDFileHandler.java +++ b/source/net/yacy/server/http/HTTPDFileHandler.java @@ -92,10 +92,11 @@ public static java.util.Hashtable respond(java.util.HashMap, serverSwitch) import java.util.zip.GZIPOutputStream; import net.yacy.cora.date.GenericFormatter; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; import net.yacy.cora.document.analysis.Classification; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.order.Digest; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.Domains; @@ -110,7 +111,6 @@ public static java.util.Hashtable respond(java.util.HashMap, serverSwitch) import net.yacy.document.parser.htmlParser; import net.yacy.document.parser.html.ContentScraper; import net.yacy.document.parser.html.ScraperInputStream; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.MemoryControl; import net.yacy.peers.Seed; @@ -1065,7 +1065,7 @@ && matchesSuffix(path, switchboard.getConfig("cgi.suffixes", null)) // "right" f // save position fis.mark(1000); // scrape document to look up charset - final ScraperInputStream htmlFilter = new ScraperInputStream(fis, "UTF-8", new DigestURI("http://localhost"), null, false, 10); + final ScraperInputStream htmlFilter = new ScraperInputStream(fis, "UTF-8", new DigestURL("http://localhost"), null, false, 10); final String charset = htmlParser.patchCharsetEncoding(htmlFilter.detectCharset()); htmlFilter.close(); if (charset != null) mimeType = mimeType + "; charset="+charset; @@ -1587,7 +1587,7 @@ private static void doURLProxy(final HashMap conProp, final Requ final Matcher m = p.matcher(sbuffer); final StringBuffer result = new StringBuffer(80); String init, url; - MultiProtocolURI target; + MultiProtocolURL target; while (m.find()) { init = null; if(m.group(1) != null) init = m.group(1); @@ -1609,7 +1609,7 @@ private static void doURLProxy(final HashMap conProp, final Requ } else if (url.startsWith("http")) { // absoulte url of form href="http://domain.com/path" if (sb.getConfig("proxyURL.rewriteURLs", "all").equals("domainlist")) { - if (sb.crawlStacker.urlInAcceptedDomain(new DigestURI(url)) != null) { + if (sb.crawlStacker.urlInAcceptedDomain(new DigestURL(url)) != null) { continue; } } @@ -1622,7 +1622,7 @@ private static void doURLProxy(final HashMap conProp, final Requ // absoulte url but same protocol of form href="//domain.com/path" final String complete_url = proxyurl.getProtocol() + ":" + url; if (sb.getConfig("proxyURL.rewriteURLs", "all").equals("domainlist")) { - if (sb.crawlStacker.urlInAcceptedDomain(new DigestURI(complete_url)) != null) { + if (sb.crawlStacker.urlInAcceptedDomain(new DigestURL(complete_url)) != null) { continue; } } @@ -1640,7 +1640,7 @@ private static void doURLProxy(final HashMap conProp, final Requ } else { // relative path of form href="relative/path" try { - target = new MultiProtocolURI("http://" + host + directory + "/" + url); + target = new MultiProtocolURL("http://" + host + directory + "/" + url); String newurl = init + "/proxy.html?url=" + target.toString(); newurl = newurl.replaceAll("\\$","\\\\\\$"); m.appendReplacement(result, newurl); diff --git a/source/net/yacy/server/http/HTTPDProxyHandler.java b/source/net/yacy/server/http/HTTPDProxyHandler.java index 07df1f8f23..ab0c5d17df 100644 --- a/source/net/yacy/server/http/HTTPDProxyHandler.java +++ b/source/net/yacy/server/http/HTTPDProxyHandler.java @@ -69,7 +69,8 @@ import java.util.logging.LogManager; import java.util.logging.Logger; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; @@ -85,7 +86,6 @@ import net.yacy.document.TextParser; import net.yacy.document.parser.html.ContentTransformer; import net.yacy.document.parser.html.Transformer; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.io.ByteCountOutputStream; import net.yacy.kelondro.util.FileUtils; import net.yacy.repository.Blacklist.BlacklistType; @@ -307,7 +307,7 @@ public static void doGet(final HashMap conProp, final RequestHea int pos=0; int port=0; - DigestURI url = null; + DigestURL url = null; try { url = HeaderFramework.getRequestURL(conProp); if (log.isFine()) log.fine(reqID +" GET "+ url); @@ -322,7 +322,7 @@ public static void doGet(final HashMap conProp, final RequestHea final String newUrl = redirectorReader.readLine(); if (!newUrl.equals("")) { try { - url = new DigestURI(newUrl); + url = new DigestURL(newUrl); } catch(final MalformedURLException e){}//just keep the old one } if (log.isFinest()) log.finest(reqID +" using redirector to "+ url); @@ -443,7 +443,7 @@ public static void doGet(final HashMap conProp, final RequestHea } } - private static void fulfillRequestFromWeb(final HashMap conProp, final DigestURI url, final RequestHeader requestHeader, final ResponseHeader cachedResponseHeader, final OutputStream respond, final ClientIdentification.Agent agent) { + private static void fulfillRequestFromWeb(final HashMap conProp, final DigestURL url, final RequestHeader requestHeader, final ResponseHeader cachedResponseHeader, final OutputStream respond, final ClientIdentification.Agent agent) { try { final boolean proxyAugmentation = sb.getConfigBool("proxyAugmentation", false); final int reqID = requestHeader.hashCode(); @@ -695,7 +695,7 @@ private static boolean hasBody(final int statusCode) { private static void fulfillRequestFromCache( final HashMap conProp, - final DigestURI url, + final DigestURL url, final RequestHeader requestHeader, final ResponseHeader cachedResponseHeader, final byte[] cacheEntry, @@ -764,7 +764,7 @@ private static void fulfillRequestFromCache( public static void doHead(final HashMap conProp, final RequestHeader requestHeader, OutputStream respond, final ClientIdentification.Agent agent) { // ResponseContainer res = null; - DigestURI url = null; + DigestURL url = null; try { final int reqID = requestHeader.hashCode(); // remembering the starting time of the request @@ -790,7 +790,7 @@ public static void doHead(final HashMap conProp, final RequestHe } try { - url = new DigestURI("http", host, port, (args == null) ? path : path + "?" + args); + url = new DigestURL("http", host, port, (args == null) ? path : path + "?" + args); } catch (final MalformedURLException e) { final String errorMsg = "ERROR: internal error with url generation: host=" + host + ", port=" + port + ", path=" + path + ", args=" + args; @@ -881,7 +881,7 @@ public static void doPost(final HashMap conProp, final RequestHe assert conProp != null : "precondition violated: conProp != null"; assert requestHeader != null : "precondition violated: requestHeader != null"; assert body != null : "precondition violated: body != null"; - DigestURI url = null; + DigestURL url = null; ByteCountOutputStream countedRespond = null; try { final int reqID = requestHeader.hashCode(); @@ -908,7 +908,7 @@ public static void doPost(final HashMap conProp, final RequestHe } try { - url = new DigestURI("http", host, port, (args == null) ? path : path + "?" + args); + url = new DigestURL("http", host, port, (args == null) ? path : path + "?" + args); } catch (final MalformedURLException e) { final String errorMsg = "ERROR: internal error with url generation: host=" + host + ", port=" + port + ", path=" + path + ", args=" + args; @@ -1358,7 +1358,7 @@ public void pleaseTerminate() { } } - private static void handleProxyException(final Exception e, final HashMap conProp, final OutputStream respond, final DigestURI url) { + private static void handleProxyException(final Exception e, final HashMap conProp, final OutputStream respond, final DigestURL url) { // this may happen if // - the targeted host does not exist // - anything with the remote server was wrong. diff --git a/source/net/yacy/server/http/HTTPDemon.java b/source/net/yacy/server/http/HTTPDemon.java index 451fd14de9..de5274283b 100644 --- a/source/net/yacy/server/http/HTTPDemon.java +++ b/source/net/yacy/server/http/HTTPDemon.java @@ -52,8 +52,9 @@ import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.Domains; @@ -65,7 +66,6 @@ import net.yacy.cora.util.NumberTools; import net.yacy.data.UserDB; import net.yacy.document.parser.html.CharacterCoding; -import net.yacy.kelondro.data.meta.DigestURI; import net.yacy.kelondro.util.FileUtils; import net.yacy.kelondro.util.MemoryControl; import net.yacy.search.Switchboard; @@ -1071,7 +1071,7 @@ else if (httpVersion.equals("HTTP/1.1") && HeaderFramework.http1_1.containsKey(I String urlString; try { - urlString = (new DigestURI((method.equals(HeaderFramework.METHOD_CONNECT)?"https":"http"), host, port, (args == null) ? path : path + "?" + args)).toString(); + urlString = (new DigestURL((method.equals(HeaderFramework.METHOD_CONNECT)?"https":"http"), host, port, (args == null) ? path : path + "?" + args)).toString(); } catch (final MalformedURLException e) { urlString = "invalid URL"; } diff --git a/source/net/yacy/server/http/ServerSideIncludes.java b/source/net/yacy/server/http/ServerSideIncludes.java index b452fa59ee..e2a848f33c 100644 --- a/source/net/yacy/server/http/ServerSideIncludes.java +++ b/source/net/yacy/server/http/ServerSideIncludes.java @@ -31,7 +31,7 @@ import java.io.OutputStream; import java.util.HashMap; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ByteBuffer; diff --git a/source/net/yacy/server/http/TemplateEngine.java b/source/net/yacy/server/http/TemplateEngine.java index 4d9bb0a878..41074eb83d 100644 --- a/source/net/yacy/server/http/TemplateEngine.java +++ b/source/net/yacy/server/http/TemplateEngine.java @@ -56,8 +56,8 @@ import java.io.OutputStream; import java.io.PushbackInputStream; -import net.yacy.cora.document.ASCII; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.util.ByteBuffer; import net.yacy.cora.util.ConcurrentLog; import net.yacy.kelondro.util.FileUtils; diff --git a/source/net/yacy/server/serverCore.java b/source/net/yacy/server/serverCore.java index 2053094b88..2f1d962f74 100644 --- a/source/net/yacy/server/serverCore.java +++ b/source/net/yacy/server/serverCore.java @@ -58,7 +58,7 @@ import javax.net.ssl.SSLSocket; import javax.net.ssl.SSLSocketFactory; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.protocol.Domains; import net.yacy.cora.util.ByteBuffer; import net.yacy.cora.util.ConcurrentLog; diff --git a/source/net/yacy/server/serverObjects.java b/source/net/yacy/server/serverObjects.java index f844d8f282..62310a801a 100644 --- a/source/net/yacy/server/serverObjects.java +++ b/source/net/yacy/server/serverObjects.java @@ -59,8 +59,8 @@ This shall speed up usage when a slow internet connection is used (dial-up) import java.util.Set; import java.util.regex.Pattern; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; +import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.RequestHeader.FileType; import net.yacy.document.parser.html.CharacterCoding; @@ -531,9 +531,9 @@ public String toString() { if (this.map.getMap().isEmpty()) return ""; final StringBuilder param = new StringBuilder(this.map.getMap().size() * 40); for (final Map.Entry entry: entrySet()) { - param.append(MultiProtocolURI.escape(entry.getKey())) + param.append(MultiProtocolURL.escape(entry.getKey())) .append('=') - .append(MultiProtocolURI.escape(entry.getValue())) + .append(MultiProtocolURL.escape(entry.getValue())) .append('&'); } param.setLength(param.length() - 1); diff --git a/source/net/yacy/utils/bitfield.java b/source/net/yacy/utils/bitfield.java index 81c1478ce2..ea541245fe 100644 --- a/source/net/yacy/utils/bitfield.java +++ b/source/net/yacy/utils/bitfield.java @@ -20,7 +20,7 @@ package net.yacy.utils; -import net.yacy.cora.document.ASCII; +import net.yacy.cora.document.encoding.ASCII; public class bitfield { diff --git a/source/net/yacy/utils/cryptbig.java b/source/net/yacy/utils/cryptbig.java index 3f356df254..cb077d29fd 100644 --- a/source/net/yacy/utils/cryptbig.java +++ b/source/net/yacy/utils/cryptbig.java @@ -55,7 +55,7 @@ import javax.crypto.spec.PBEKeySpec; import javax.crypto.spec.PBEParameterSpec; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.Digest; import net.yacy.cora.util.CommonPattern; diff --git a/source/net/yacy/utils/gzip.java b/source/net/yacy/utils/gzip.java index 3fefbbec07..11b01fef68 100644 --- a/source/net/yacy/utils/gzip.java +++ b/source/net/yacy/utils/gzip.java @@ -38,7 +38,7 @@ import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import net.yacy.cora.document.UTF8; +import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.util.ConcurrentLog; diff --git a/source/net/yacy/utils/loaderThreads.java b/source/net/yacy/utils/loaderThreads.java index 574e7210f4..0deaa99602 100644 --- a/source/net/yacy/utils/loaderThreads.java +++ b/source/net/yacy/utils/loaderThreads.java @@ -27,9 +27,9 @@ import java.util.HashMap; import java.util.Map; +import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.http.ProxySettings; -import net.yacy.kelondro.data.meta.DigestURI; public class loaderThreads { @@ -60,7 +60,7 @@ public loaderThreads( this.failed = 0; } - public void newThread(final String name, final DigestURI url, final loaderProcess process, final ClientIdentification.Agent agent) { + public void newThread(final String name, final DigestURL url, final loaderProcess process, final ClientIdentification.Agent agent) { final Thread t = new loaderThread(url, process, agent); this.threads.put(name, t); t.start(); @@ -103,14 +103,14 @@ public Exception threadError(final String name) { } protected class loaderThread extends Thread { - private final DigestURI url; + private final DigestURL url; private Exception error; private final loaderProcess process; private byte[] page; private boolean loaded; final ClientIdentification.Agent agent; - public loaderThread(final DigestURI url, final loaderProcess process, final ClientIdentification.Agent agent) { + public loaderThread(final DigestURL url, final loaderProcess process, final ClientIdentification.Agent agent) { this.url = url; this.process = process; this.error = null;