diff --git a/source/net/yacy/kelondro/data/meta/URIMetadataRow.java b/source/net/yacy/kelondro/data/meta/URIMetadataRow.java index 9e77f7e254..c1f606ab2b 100644 --- a/source/net/yacy/kelondro/data/meta/URIMetadataRow.java +++ b/source/net/yacy/kelondro/data/meta/URIMetadataRow.java @@ -9,7 +9,7 @@ // $LastChangedBy$ // // LICENSE -// +// // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or @@ -46,9 +46,8 @@ import net.yacy.kelondro.order.Digest; import net.yacy.kelondro.order.NaturalOrder; import net.yacy.kelondro.util.ByteBuffer; -import net.yacy.kelondro.util.kelondroException; import net.yacy.kelondro.util.MapTools; - +import net.yacy.kelondro.util.kelondroException; import de.anomic.crawler.retrieval.Request; import de.anomic.search.QueryParams; import de.anomic.tools.crypt; @@ -56,7 +55,7 @@ public class URIMetadataRow implements URIMetadata { // this object stores attributes for URL entries - + public static final Row rowdef = new Row( "String hash-12, " + // the url's hash "String comp-360, " + // components: the url, description, author, tags and publisher @@ -70,15 +69,15 @@ public class URIMetadataRow implements URIMetadata { "byte[] dt-1, " + // doctype, taken from extension or any other heuristic "Bitfield flags-4, " + // flags; any stuff (see Word-Entity definition) "byte[] lang-2, " + // language - "Cardinal llocal-2 {b256}, " + // # of outlinks to same domain; for video and image: width + "Cardinal llocal-2 {b256}, " + // # of outlinks to same domain; for video and image: width "Cardinal lother-2 {b256}, " + // # of outlinks to outside domain; for video and image: height "Cardinal limage-2 {b256}, " + // # of embedded image links "Cardinal laudio-2 {b256}, " + // # of embedded audio links; for audio: track number; for video: number of audio tracks "Cardinal lvideo-2 {b256}, " + // # of embedded video links "Cardinal lapp-2 {b256}", // # of embedded links to applications Base64Order.enhancedCoder - ); - + ); + /* =========================================================================== * Constants to access the various columns of an URL entry * =========================================================================== */ @@ -100,13 +99,13 @@ public class URIMetadataRow implements URIMetadata { private static final int col_laudio = 15; // # of embedded audio links; for audio: track number; for video: number of audio tracks private static final int col_lvideo = 16; // # of embedded video links private static final int col_lapp = 17; // # of embedded links to applications - + private final Row.Entry entry; private final String snippet; private WordReferenceVars word; // this is only used if the url is transported via remote search requests private final long ranking; // during generation of a search result this value is set private Components comp; - + public URIMetadataRow() { // create a dummy entry, good to produce poison objects this.entry = rowdef.newEntry(); @@ -115,14 +114,14 @@ public URIMetadataRow() { this.ranking = 0; this.comp = null; } - + public URIMetadataRow( final DigestURI url, final String dc_title, final String dc_creator, final String dc_subject, final String dc_publisher, - final float lon, final float lat, // decimal degrees as in WGS84; if unknown both values may be 0.0f; + final float lon, final float lat, // decimal degrees as in WGS84; if unknown both values may be 0.0f; final Date mod, final Date load, final Date fresh, @@ -173,14 +172,14 @@ private void encodeDate(final int col, final Date d) { } private Date decodeDate(final int col) { - long t = this.entry.getColLong(col); + final long t = this.entry.getColLong(col); /*if (t < 14600) */return new Date(86400000L * t); // time was stored as number of days since epoch /* if (t < 350400) return new Date(3600000L * t); // hours since epoch if (t < 21024000) return new Date(60000L * t); // minutes since epoch */ } - + public static byte[] encodeComp( final DigestURI url, final String dc_title, @@ -198,7 +197,7 @@ public static byte[] encodeComp( if (lon == 0.0f && lat == 0.0f) s.append(10); else s.append(Float.toString(lat)).append(',').append(Float.toString(lon)).append(10); return UTF8.getBytes(s.toString()); } - + public URIMetadataRow(final Row.Entry entry, final WordReferenceVars searchedWord, final long ranking) { this.entry = entry; this.snippet = null; @@ -223,14 +222,14 @@ public URIMetadataRow(final Properties prop) { String dc_publisher = crypt.simpleDecode(prop.getProperty("publisher", ""), null); if (dc_publisher == null) dc_publisher = ""; String lons = crypt.simpleDecode(prop.getProperty("lon", "0.0"), null); if (lons == null) lons = "0.0"; String lats = crypt.simpleDecode(prop.getProperty("lat", "0.0"), null); if (lats == null) lats = "0.0"; - + this.entry = rowdef.newEntry(); this.entry.setCol(col_hash, url.hash()); // FIXME potential null pointer access this.entry.setCol(col_comp, encodeComp(url, descr, dc_creator, tags, dc_publisher, Float.parseFloat(lats), Float.parseFloat(lons))); // create new formatters to make concurrency possible - GenericFormatter formatter = new GenericFormatter(GenericFormatter.FORMAT_SHORT_DAY, GenericFormatter.time_minute); - + final GenericFormatter formatter = new GenericFormatter(GenericFormatter.FORMAT_SHORT_DAY, GenericFormatter.time_minute); + try { encodeDate(col_mod, formatter.parse(prop.getProperty("mod", "20000101"))); } catch (final ParseException e) { @@ -250,7 +249,7 @@ public URIMetadataRow(final Properties prop) { this.entry.setCol(col_md5, Digest.decodeHex(prop.getProperty("md5", ""))); this.entry.setCol(col_size, Integer.parseInt(prop.getProperty("size", "0"))); this.entry.setCol(col_wc, Integer.parseInt(prop.getProperty("wc", "0"))); - String dt = prop.getProperty("dt", "t"); + final String dt = prop.getProperty("dt", "t"); this.entry.setCol(col_dt, dt.length() > 0 ? new byte[]{(byte) dt.charAt(0)} : new byte[]{(byte) 't'}); final String flags = prop.getProperty("flags", "AAAAAA"); this.entry.setCol(col_flags, (flags.length() > 6) ? QueryParams.empty_constraint.bytes() : (new Bitfield(4, flags)).bytes()); @@ -285,14 +284,14 @@ public static URIMetadataRow importEntry(final String propStr) { private StringBuilder corePropList() { // generate a parseable string; this is a simple property-list - final Components metadata = this.metadata(); + final Components metadata = metadata(); final StringBuilder s = new StringBuilder(300); if (metadata == null) return null; //System.out.println("author=" + comp.author()); - + // create new formatters to make concurrency possible - GenericFormatter formatter = new GenericFormatter(GenericFormatter.FORMAT_SHORT_DAY, GenericFormatter.time_minute); - + final GenericFormatter formatter = new GenericFormatter(GenericFormatter.FORMAT_SHORT_DAY, GenericFormatter.time_minute); + try { s.append("hash=").append(ASCII.String(hash())); assert (s.toString().indexOf(0) < 0); @@ -342,15 +341,15 @@ private StringBuilder corePropList() { assert (s.toString().indexOf(0) < 0); s.append(",lapp=").append(lapp()); assert (s.toString().indexOf(0) < 0); - + if (this.word != null) { // append also word properties - s.append(",wi=").append(Base64Order.enhancedCoder.encodeString(word.toPropertyForm())); + s.append(",wi=").append(Base64Order.enhancedCoder.encodeString(this.word.toPropertyForm())); } assert (s.toString().indexOf(0) < 0); return s; - } catch (final Exception e) { + } catch (final Throwable e) { // serverLog.logFailure("plasmaLURL.corePropList", e.getMessage()); // if (moddate == null) serverLog.logFailure("plasmaLURL.corePropList", "moddate=null"); // if (loaddate == null) serverLog.logFailure("plasmaLURL.corePropList", "loaddate=null"); @@ -374,13 +373,13 @@ public byte[] hash() { public long ranking() { return this.ranking; } - + public Components metadata() { // avoid double computation of metadata elements if (this.comp != null) return this.comp; // parse elements from comp field; - byte[] c = this.entry.getColBytes(col_comp, true); - List cl = ByteBuffer.split(c, (byte) 10); + final byte[] c = this.entry.getColBytes(col_comp, true); + final List cl = ByteBuffer.split(c, (byte) 10); this.comp = new Components( (cl.size() > 0) ? UTF8.String(cl.get(0)) : "", hash(), @@ -391,7 +390,7 @@ public Components metadata() { (cl.size() > 5) ? UTF8.String(cl.get(5)) : ""); return this.comp; } - + public Date moddate() { return decodeDate(col_mod); } @@ -407,7 +406,7 @@ public Date freshdate() { public byte[] referrerHash() { // return the creator's hash or null if there is none // FIXME: There seem to be some malformed entries in the databasees like "null\0\0\0\0\0\0\0\0" - final byte[] r = entry.getColBytes(col_referrer, true); + final byte[] r = this.entry.getColBytes(col_referrer, true); if (r != null) { int i = r.length; while (i > 0) { @@ -419,11 +418,11 @@ public byte[] referrerHash() { public String md5() { // returns the md5 in hex representation - return Digest.encodeHex(entry.getColBytes(col_md5, true)); + return Digest.encodeHex(this.entry.getColBytes(col_md5, true)); } public char doctype() { - return (char) entry.getColByte(col_dt); + return (char) this.entry.getColByte(col_dt); } public byte[] language() { @@ -465,15 +464,15 @@ public int lvideo() { public int lapp() { return (int) this.entry.getColLong(col_lapp); } - + public String snippet() { // the snippet may appear here if the url was transported in a remote search // it will not be saved anywhere, but can only be requested here - return snippet; + return this.snippet; } public WordReferenceVars word() { - return word; + return this.word; } public boolean isOlder(final URIMetadata other) { @@ -507,20 +506,20 @@ public String toString(final String snippet) { public Request toBalancerEntry(final String initiatorHash) { return new Request( - ASCII.getBytes(initiatorHash), - metadata().url(), - referrerHash(), + ASCII.getBytes(initiatorHash), + metadata().url(), + referrerHash(), metadata().dc_title(), moddate(), null, - 0, - 0, + 0, + 0, 0, 0); } - + /** - * @return the object as String.
+ * @return the object as String.
* This e.g. looks like this: * 
{hash=jmqfMk7Y3NKw,referrer=------------,mod=20050610,load=20051003,size=51666,wc=1392,cc=0,local=true,q=AEn,dt=h,lang=uk,url=b|aHR0cDovL3d3dy50cmFuc3BhcmVuY3kub3JnL3N1cnZleXMv,descr=b|S25vd2xlZGdlIENlbnRyZTogQ29ycnVwdGlvbiBTdXJ2ZXlzIGFuZCBJbmRpY2Vz}
*/ @@ -535,14 +534,14 @@ public String toString() { return core.toString(); //return "{" + core + "}"; } - + public class Components { private DigestURI url; private String urlRaw; private byte[] urlHash; private final String dc_title, dc_creator, dc_subject, dc_publisher; private final String latlon; // a comma-separated tuple as "," where the coordinates are given as WGS84 spatial coordinates in decimal degrees - + public Components( final String urlRaw, final byte[] urlhash, @@ -560,7 +559,7 @@ public Components( this.dc_publisher = publisher; this.latlon = latlon; } - public boolean matches(Pattern matcher) { + public boolean matches(final Pattern matcher) { if (this.urlRaw != null) return matcher.matcher(this.urlRaw.toLowerCase()).matches(); if (this.url != null) return matcher.matcher(this.url.toNormalform(true, true).toLowerCase()).matches(); return false; @@ -569,7 +568,7 @@ public DigestURI url() { if (this.url == null) { try { this.url = new DigestURI(this.urlRaw, this.urlHash); - } catch (MalformedURLException e) { + } catch (final MalformedURLException e) { this.url = null; } this.urlRaw = null; @@ -582,14 +581,14 @@ public DigestURI url() { public String dc_publisher() { return this.dc_publisher; } public String dc_subject() { return this.dc_subject; } public float lat() { - if (latlon == null || latlon.length() == 0) return 0.0f; - int p = latlon.indexOf(','); - return p < 0 ? 0.0f : Float.parseFloat(latlon.substring(0, p)); + if (this.latlon == null || this.latlon.length() == 0) return 0.0f; + final int p = this.latlon.indexOf(','); + return p < 0 ? 0.0f : Float.parseFloat(this.latlon.substring(0, p)); } public float lon() { - if (latlon == null || latlon.length() == 0) return 0.0f; - int p = latlon.indexOf(','); - return p < 0 ? 0.0f : Float.parseFloat(latlon.substring(p + 1)); + if (this.latlon == null || this.latlon.length() == 0) return 0.0f; + final int p = this.latlon.indexOf(','); + return p < 0 ? 0.0f : Float.parseFloat(this.latlon.substring(p + 1)); } } } diff --git a/source/net/yacy/kelondro/index/RowSet.java b/source/net/yacy/kelondro/index/RowSet.java index 9a1fe56e74..5e838a3f89 100644 --- a/source/net/yacy/kelondro/index/RowSet.java +++ b/source/net/yacy/kelondro/index/RowSet.java @@ -459,13 +459,13 @@ protected final static RowSet mergeEnum(final RowCollection c0, final RowCollect final RowSet r = new RowSet(c0.rowdef, c0.size() + c1.size()); try { c0.sort(); - } catch (final Exception e) { + } catch (final Throwable e) { Log.logSevere("RowSet", "collection corrupted. cleaned. " + e.getMessage(), e); c0.clear(); } try { c1.sort(); - } catch (final Exception e) { + } catch (final Throwable e) { Log.logSevere("RowSet", "collection corrupted. cleaned. " + e.getMessage(), e); c1.clear(); } diff --git a/source/net/yacy/kelondro/order/Base64Order.java b/source/net/yacy/kelondro/order/Base64Order.java index 15c5f91fa1..094f50675d 100644 --- a/source/net/yacy/kelondro/order/Base64Order.java +++ b/source/net/yacy/kelondro/order/Base64Order.java @@ -409,7 +409,7 @@ public final int compare(final byte[] a, final byte[] b) { ((this.zero == null) ? compares(a, b) : compare0(a, 0, b, 0, a.length)) : ((this.zero == null) ? compares(b, a) : compare0(b, 0, a, 0, a.length)); - } catch (final Exception e) { + } catch (final Throwable e) { // if a or b is not well-formed, an ArrayIndexOutOfBoundsException may occur // in that case we don't want that the exception makes databse functions // unusable and effective creates a showstopper. In such cases we apply @@ -435,7 +435,7 @@ public final int compare(final byte[] a, final int aoffset, final byte[] b, fina compare0(a, aoffset, b, boffset, length) : compare0(b, boffset, a, aoffset, length); - } catch (final Exception e) { + } catch (final Throwable e) { // same handling as in simple compare method above final boolean wfa = wellformed(a, aoffset, length); final boolean wfb = wellformed(b, boffset, length); diff --git a/source/net/yacy/kelondro/rwi/IODispatcher.java b/source/net/yacy/kelondro/rwi/IODispatcher.java index 59166081ef..04512256b9 100644 --- a/source/net/yacy/kelondro/rwi/IODispatcher.java +++ b/source/net/yacy/kelondro/rwi/IODispatcher.java @@ -162,7 +162,7 @@ public void run() { } catch (final InterruptedException e) { Log.logSevere("IODispatcher", "main run job was interrupted (1)", e); Log.logException(e); - } catch (final Exception e) { + } catch (final Throwable e) { Log.logSevere("IODispatcher", "main run job had errors (1), dump to " + f + " failed.", e); Log.logException(e); } @@ -181,7 +181,7 @@ public void run() { } catch (final InterruptedException e) { Log.logSevere("IODispatcher", "main run job was interrupted (2)", e); Log.logException(e); - } catch (final Exception e) { + } catch (final Throwable e) { if (f2 == null) { Log.logSevere("IODispatcher", "main run job had errors (2), dump to " + f + " failed. Input file is " + f1, e); } else { @@ -200,12 +200,12 @@ public void run() { Log.logSevere("IODispatcher", "main loop in bad state, dumpQueue.size() = " + this.dumpQueue.size() + ", mergeQueue.size() = " + this.mergeQueue.size() + ", controlQueue.availablePermits() = " + this.controlQueue.availablePermits()); assert false : "this process statt should not be reached"; // this should never happen - } catch (final Exception e) { + } catch (final Throwable e) { Log.logSevere("IODispatcher", "main run job failed (X)", e); Log.logException(e); } Log.logInfo("IODispatcher", "loop terminated"); - } catch (final Exception e) { + } catch (final Throwable e) { Log.logSevere("IODispatcher", "main run job failed (4)", e); Log.logException(e); } finally { diff --git a/source/net/yacy/kelondro/rwi/IndexCell.java b/source/net/yacy/kelondro/rwi/IndexCell.java index 1a7a4a9d04..9bf68844e6 100644 --- a/source/net/yacy/kelondro/rwi/IndexCell.java +++ b/source/net/yacy/kelondro/rwi/IndexCell.java @@ -111,7 +111,7 @@ public void run() { while (IndexCell.this.cleanupShallRun) { try { cleanCache(); - } catch (final Exception e) { + } catch (final Throwable e) { Log.logException(e); } try { Thread.sleep(3000); } catch (final InterruptedException e) {} @@ -147,7 +147,7 @@ private void cleanCache() { // dump the buffer IndexCell.this.merger.dump(ramdump, dumpFile, IndexCell.this.array); IndexCell.this.lastDump = System.currentTimeMillis(); - } catch (final Exception e) { + } catch (final Throwable e) { // catch all exceptions Log.logException(e); } @@ -162,7 +162,7 @@ private void cleanCache() { IndexCell.this.lastCleanup = System.currentTimeMillis(); // set time to prevent that this is called to soon again IndexCell.this.shrink(IndexCell.this.targetFileSize, IndexCell.this.maxFileSize); IndexCell.this.lastCleanup = System.currentTimeMillis(); // set again to mark end of procedure - } catch (final Exception e) { + } catch (final Throwable e) { // catch all exceptions Log.logException(e); } @@ -263,7 +263,7 @@ public int count(final byte[] termHash) { // read fresh values from file try { countFile = this.array.count(termHash); - } catch (final Exception e) { + } catch (final Throwable e) { Log.logException(e); } assert countFile >= 0; diff --git a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java index 079aeac193..c492ffee14 100644 --- a/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java +++ b/source/net/yacy/kelondro/rwi/ReferenceContainerArray.java @@ -153,7 +153,7 @@ public boolean hasNext() { public ReferenceContainer next() { if (this.iterator.hasNext()) try { return get(this.iterator.next()); - } catch (final Exception e) { + } catch (final Throwable e) { Log.logException(e); return null; } @@ -164,7 +164,7 @@ public ReferenceContainer next() { try { this.iterator = ReferenceContainerArray.this.array.keys(true, null); return get(this.iterator.next()); - } catch (final Exception e) { + } catch (final Throwable e) { Log.logException(e); return null; } @@ -225,7 +225,7 @@ public Rating next() { if (this.iterator.hasNext()) try { reference = this.iterator.next(); return new Rating(reference, count(reference)); - } catch (final Exception e) { + } catch (final Throwable e) { Log.logException(e); return null; } @@ -237,7 +237,7 @@ public Rating next() { this.iterator = ReferenceContainerArray.this.array.keys(true, null); reference = this.iterator.next(); return new Rating(reference, count(reference)); - } catch (final Exception e) { + } catch (final Throwable e) { Log.logException(e); return null; }