From 575fb022ee465487bb431c70f91995de25d755ea Mon Sep 17 00:00:00 2001 From: Rolf Date: Sun, 18 Mar 2012 19:11:56 -0400 Subject: [PATCH] Three major issues fixed... 1. HTTP protocol uses Base64 to encode the MD5 sum, not hex - fixed. 2. There is a bug in URLConnection.getLastModified() in that it appears not to use the TimeZone of the input date to process the time. Work around that. 3 Windows does not like ':' as part of a filename, change the cache path to use '~' instead of ':' This commit adds the BSD-licensed Base64.java code by Mikael Grev, MiG InfoCom AB. --- src/main/net/tuis/resolver/Base64.Changes.txt | 9 + src/main/net/tuis/resolver/Base64.java | 576 ++++++++++++++++++ .../tuis/resolver/CachedHTTPRepository.java | 46 +- 3 files changed, 612 insertions(+), 19 deletions(-) create mode 100644 src/main/net/tuis/resolver/Base64.Changes.txt create mode 100644 src/main/net/tuis/resolver/Base64.java diff --git a/src/main/net/tuis/resolver/Base64.Changes.txt b/src/main/net/tuis/resolver/Base64.Changes.txt new file mode 100644 index 0000000..c530182 --- /dev/null +++ b/src/main/net/tuis/resolver/Base64.Changes.txt @@ -0,0 +1,9 @@ +Changelog 2.1 > 2.2 + +* Added a "fast" decode method that is a little more than two times faster + than the normal one but has some moderate restriction on input. + +* Normal decode now handles negative bytes and just disregard them as + specified in RFC 2045. + +* Some minor enhancements \ No newline at end of file diff --git a/src/main/net/tuis/resolver/Base64.java b/src/main/net/tuis/resolver/Base64.java new file mode 100644 index 0000000..a3d3576 --- /dev/null +++ b/src/main/net/tuis/resolver/Base64.java @@ -0,0 +1,576 @@ +// RGL - changed package. +package net.tuis.resolver; + +import java.util.Arrays; + +/** A very fast and memory efficient class to encode and decode to and from BASE64 in full accordance + * with RFC 2045.

+ * On Windows XP sp1 with 1.4.2_04 and later ;), this encoder and decoder is about 10 times faster + * on small arrays (10 - 1000 bytes) and 2-3 times as fast on larger arrays (10000 - 1000000 bytes) + * compared to sun.misc.Encoder()/Decoder().

+ * + * On byte arrays the encoder is about 20% faster than Jakarta Commons Base64 Codec for encode and + * about 50% faster for decoding large arrays. This implementation is about twice as fast on very small + * arrays (< 30 bytes). If source/destination is a String this + * version is about three times as fast due to the fact that the Commons Codec result has to be recoded + * to a String from byte[], which is very expensive.

+ * + * This encode/decode algorithm doesn't create any temporary arrays as many other codecs do, it only + * allocates the resulting array. This produces less garbage and it is possible to handle arrays twice + * as large as algorithms that create a temporary array. (E.g. Jakarta Commons Codec). It is unknown + * whether Sun's sun.misc.Encoder()/Decoder() produce temporary arrays but since performance + * is quite low it probably does.

+ * + * The encoder produces the same output as the Sun one except that the Sun's encoder appends + * a trailing line separator if the last character isn't a pad. Unclear why but it only adds to the + * length and is probably a side effect. Both are in conformance with RFC 2045 though.
+ * Commons codec seem to always att a trailing line separator.

+ * + * Note! + * The encode/decode method pairs (types) come in three versions with the exact same algorithm and + * thus a lot of code redundancy. This is to not create any temporary arrays for transcoding to/from different + * format types. The methods not used can simply be commented out.

+ * + * There is also a "fast" version of all decode methods that works the same way as the normal ones, but + * har a few demands on the decoded input. Normally though, these fast verions should be used if the source if + * the input is known and it hasn't bee tampered with.

+ * + * If you find the code useful or you find a bug, please send me a note at base64 @ miginfocom . com. + * + * Licence (BSD): + * ============== + * + * Copyright (c) 2004, Mikael Grev, MiG InfoCom AB. (base64 @ miginfocom . com) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this list + * of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or other + * materials provided with the distribution. + * Neither the name of the MiG InfoCom AB nor the names of its contributors may be + * used to endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, + * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY + * OF SUCH DAMAGE. + * + * @version 2.2 + * @author Mikael Grev + * Date: 2004-aug-02 + * Time: 11:31:11 + */ + +public class Base64 +{ + private static final char[] CA = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".toCharArray(); + private static final int[] IA = new int[256]; + static { + Arrays.fill(IA, -1); + for (int i = 0, iS = CA.length; i < iS; i++) + IA[CA[i]] = i; + IA['='] = 0; + } + + // **************************************************************************************** + // * char[] version + // **************************************************************************************** + + /** Encodes a raw byte array into a BASE64 char[] representation i accordance with RFC 2045. + * @param sArr The bytes to convert. If null or length 0 an empty array will be returned. + * @param lineSep Optional "\r\n" after 76 characters, unless end of file.
+ * No line separator will be in breach of RFC 2045 which specifies max 76 per line but will be a + * little faster. + * @return A BASE64 encoded array. Never null. + */ + public final static char[] encodeToChar(byte[] sArr, boolean lineSep) + { + // Check special case + int sLen = sArr != null ? sArr.length : 0; + if (sLen == 0) + return new char[0]; + + int eLen = (sLen / 3) * 3; // Length of even 24-bits. + int cCnt = ((sLen - 1) / 3 + 1) << 2; // Returned character count + int dLen = cCnt + (lineSep ? (cCnt - 1) / 76 << 1 : 0); // Length of returned array + char[] dArr = new char[dLen]; + + // Encode even 24-bits + for (int s = 0, d = 0, cc = 0; s < eLen;) { + // Copy next three bytes into lower 24 bits of int, paying attension to sign. + int i = (sArr[s++] & 0xff) << 16 | (sArr[s++] & 0xff) << 8 | (sArr[s++] & 0xff); + + // Encode the int into four chars + dArr[d++] = CA[(i >>> 18) & 0x3f]; + dArr[d++] = CA[(i >>> 12) & 0x3f]; + dArr[d++] = CA[(i >>> 6) & 0x3f]; + dArr[d++] = CA[i & 0x3f]; + + // Add optional line separator + if (lineSep && ++cc == 19 && d < dLen - 2) { + dArr[d++] = '\r'; + dArr[d++] = '\n'; + cc = 0; + } + } + + // Pad and encode last bits if source isn't even 24 bits. + int left = sLen - eLen; // 0 - 2. + if (left > 0) { + // Prepare the int + int i = ((sArr[eLen] & 0xff) << 10) | (left == 2 ? ((sArr[sLen - 1] & 0xff) << 2) : 0); + + // Set last four chars + dArr[dLen - 4] = CA[i >> 12]; + dArr[dLen - 3] = CA[(i >>> 6) & 0x3f]; + dArr[dLen - 2] = left == 2 ? CA[i & 0x3f] : '='; + dArr[dLen - 1] = '='; + } + return dArr; + } + + /** Decodes a BASE64 encoded char array. All illegal characters will be ignored and can handle both arrays with + * and without line separators. + * @param sArr The source array. null or length 0 will return an empty array. + * @return The decoded array of bytes. May be of length 0. Will be null if the legal characters + * (including '=') isn't divideable by 4. (I.e. definitely corrupted). + */ + public final static byte[] decode(char[] sArr) + { + // Check special case + int sLen = sArr != null ? sArr.length : 0; + if (sLen == 0) + return new byte[0]; + + // Count illegal characters (including '\r', '\n') to know what size the returned array will be, + // so we don't have to reallocate & copy it later. + int sepCnt = 0; // Number of separator characters. (Actually illegal characters, but that's a bonus...) + for (int i = 0; i < sLen; i++) // If input is "pure" (I.e. no line separators or illegal chars) base64 this loop can be commented out. + if (IA[sArr[i]] < 0) + sepCnt++; + + // Check so that legal chars (including '=') are evenly divideable by 4 as specified in RFC 2045. + if ((sLen - sepCnt) % 4 != 0) + return null; + + int pad = 0; + for (int i = sLen; i > 1 && IA[sArr[--i]] <= 0;) + if (sArr[i] == '=') + pad++; + + int len = ((sLen - sepCnt) * 6 >> 3) - pad; + + byte[] dArr = new byte[len]; // Preallocate byte[] of exact length + + for (int s = 0, d = 0; d < len;) { + // Assemble three bytes into an int from four "valid" characters. + int i = 0; + for (int j = 0; j < 4; j++) { // j only increased if a valid char was found. + int c = IA[sArr[s++]]; + if (c >= 0) + i |= c << (18 - j * 6); + else + j--; + } + // Add the bytes + dArr[d++] = (byte) (i >> 16); + if (d < len) { + dArr[d++]= (byte) (i >> 8); + if (d < len) + dArr[d++] = (byte) i; + } + } + return dArr; + } + + /** Decodes a BASE64 encoded char array that is known to be resonably well formatted. The method is about twice as + * fast as {@link #decode(char[])}. The preconditions are:
+ * + The array must have a line length of 76 chars OR no line separators at all (one line).
+ * + Line separator must be "\r\n", as specified in RFC 2045 + * + The array must not contain illegal characters within the encoded string
+ * + The array CAN have illegal characters at the beginning and end, those will be dealt with appropriately.
+ * @param sArr The source array. Length 0 will return an empty array. null will throw an exception. + * @return The decoded array of bytes. May be of length 0. + */ + public final static byte[] decodeFast(char[] sArr) + { + // Check special case + int sLen = sArr.length; + if (sLen == 0) + return new byte[0]; + + int sIx = 0, eIx = sLen - 1; // Start and end index after trimming. + + // Trim illegal chars from start + while (sIx < eIx && IA[sArr[sIx]] < 0) + sIx++; + + // Trim illegal chars from end + while (eIx > 0 && IA[sArr[eIx]] < 0) + eIx--; + + // get the padding count (=) (0, 1 or 2) + int pad = sArr[eIx] == '=' ? (sArr[eIx - 1] == '=' ? 2 : 1) : 0; // Count '=' at end. + int cCnt = eIx - sIx + 1; // Content count including possible separators + int sepCnt = sLen > 76 ? (sArr[76] == '\r' ? cCnt / 78 : 0) << 1 : 0; + + int len = ((cCnt - sepCnt) * 6 >> 3) - pad; // The number of decoded bytes + byte[] dArr = new byte[len]; // Preallocate byte[] of exact length + + // Decode all but the last 0 - 2 bytes. + int d = 0; + for (int cc = 0, eLen = (len / 3) * 3; d < eLen;) { + // Assemble three bytes into an int from four "valid" characters. + int i = IA[sArr[sIx++]] << 18 | IA[sArr[sIx++]] << 12 | IA[sArr[sIx++]] << 6 | IA[sArr[sIx++]]; + + // Add the bytes + dArr[d++] = (byte) (i >> 16); + dArr[d++] = (byte) (i >> 8); + dArr[d++] = (byte) i; + + // If line separator, jump over it. + if (sepCnt > 0 && ++cc == 19) { + sIx += 2; + cc = 0; + } + } + + if (d < len) { + // Decode last 1-3 bytes (incl '=') into 1-3 bytes + int i = 0; + for (int j = 0; sIx <= eIx - pad; j++) + i |= IA[sArr[sIx++]] << (18 - j * 6); + + for (int r = 16; d < len; r -= 8) + dArr[d++] = (byte) (i >> r); + } + + return dArr; + } + + // **************************************************************************************** + // * byte[] version + // **************************************************************************************** + + /** Encodes a raw byte array into a BASE64 byte[] representation i accordance with RFC 2045. + * @param sArr The bytes to convert. If null or length 0 an empty array will be returned. + * @param lineSep Optional "\r\n" after 76 characters, unless end of file.
+ * No line separator will be in breach of RFC 2045 which specifies max 76 per line but will be a + * little faster. + * @return A BASE64 encoded array. Never null. + */ + public final static byte[] encodeToByte(byte[] sArr, boolean lineSep) + { + // Check special case + int sLen = sArr != null ? sArr.length : 0; + if (sLen == 0) + return new byte[0]; + + int eLen = (sLen / 3) * 3; // Length of even 24-bits. + int cCnt = ((sLen - 1) / 3 + 1) << 2; // Returned character count + int dLen = cCnt + (lineSep ? (cCnt - 1) / 76 << 1 : 0); // Length of returned array + byte[] dArr = new byte[dLen]; + + // Encode even 24-bits + for (int s = 0, d = 0, cc = 0; s < eLen;) { + // Copy next three bytes into lower 24 bits of int, paying attension to sign. + int i = (sArr[s++] & 0xff) << 16 | (sArr[s++] & 0xff) << 8 | (sArr[s++] & 0xff); + + // Encode the int into four chars + dArr[d++] = (byte) CA[(i >>> 18) & 0x3f]; + dArr[d++] = (byte) CA[(i >>> 12) & 0x3f]; + dArr[d++] = (byte) CA[(i >>> 6) & 0x3f]; + dArr[d++] = (byte) CA[i & 0x3f]; + + // Add optional line separator + if (lineSep && ++cc == 19 && d < dLen - 2) { + dArr[d++] = '\r'; + dArr[d++] = '\n'; + cc = 0; + } + } + + // Pad and encode last bits if source isn't an even 24 bits. + int left = sLen - eLen; // 0 - 2. + if (left > 0) { + // Prepare the int + int i = ((sArr[eLen] & 0xff) << 10) | (left == 2 ? ((sArr[sLen - 1] & 0xff) << 2) : 0); + + // Set last four chars + dArr[dLen - 4] = (byte) CA[i >> 12]; + dArr[dLen - 3] = (byte) CA[(i >>> 6) & 0x3f]; + dArr[dLen - 2] = left == 2 ? (byte) CA[i & 0x3f] : (byte) '='; + dArr[dLen - 1] = '='; + } + return dArr; + } + + /** Decodes a BASE64 encoded byte array. All illegal characters will be ignored and can handle both arrays with + * and without line separators. + * @param sArr The source array. Length 0 will return an empty array. null will throw an exception. + * @return The decoded array of bytes. May be of length 0. Will be null if the legal characters + * (including '=') isn't divideable by 4. (I.e. definitely corrupted). + */ + public final static byte[] decode(byte[] sArr) + { + // Check special case + int sLen = sArr.length; + + // Count illegal characters (including '\r', '\n') to know what size the returned array will be, + // so we don't have to reallocate & copy it later. + int sepCnt = 0; // Number of separator characters. (Actually illegal characters, but that's a bonus...) + for (int i = 0; i < sLen; i++) // If input is "pure" (I.e. no line separators or illegal chars) base64 this loop can be commented out. + if (IA[sArr[i] & 0xff] < 0) + sepCnt++; + + // Check so that legal chars (including '=') are evenly divideable by 4 as specified in RFC 2045. + if ((sLen - sepCnt) % 4 != 0) + return null; + + int pad = 0; + for (int i = sLen; i > 1 && IA[sArr[--i] & 0xff] <= 0;) + if (sArr[i] == '=') + pad++; + + int len = ((sLen - sepCnt) * 6 >> 3) - pad; + + byte[] dArr = new byte[len]; // Preallocate byte[] of exact length + + for (int s = 0, d = 0; d < len;) { + // Assemble three bytes into an int from four "valid" characters. + int i = 0; + for (int j = 0; j < 4; j++) { // j only increased if a valid char was found. + int c = IA[sArr[s++] & 0xff]; + if (c >= 0) + i |= c << (18 - j * 6); + else + j--; + } + + // Add the bytes + dArr[d++] = (byte) (i >> 16); + if (d < len) { + dArr[d++]= (byte) (i >> 8); + if (d < len) + dArr[d++] = (byte) i; + } + } + + return dArr; + } + + + /** Decodes a BASE64 encoded byte array that is known to be resonably well formatted. The method is about twice as + * fast as {@link #decode(byte[])}. The preconditions are:
+ * + The array must have a line length of 76 chars OR no line separators at all (one line).
+ * + Line separator must be "\r\n", as specified in RFC 2045 + * + The array must not contain illegal characters within the encoded string
+ * + The array CAN have illegal characters at the beginning and end, those will be dealt with appropriately.
+ * @param sArr The source array. Length 0 will return an empty array. null will throw an exception. + * @return The decoded array of bytes. May be of length 0. + */ + public final static byte[] decodeFast(byte[] sArr) + { + // Check special case + int sLen = sArr.length; + if (sLen == 0) + return new byte[0]; + + int sIx = 0, eIx = sLen - 1; // Start and end index after trimming. + + // Trim illegal chars from start + while (sIx < eIx && IA[sArr[sIx] & 0xff] < 0) + sIx++; + + // Trim illegal chars from end + while (eIx > 0 && IA[sArr[eIx] & 0xff] < 0) + eIx--; + + // get the padding count (=) (0, 1 or 2) + int pad = sArr[eIx] == '=' ? (sArr[eIx - 1] == '=' ? 2 : 1) : 0; // Count '=' at end. + int cCnt = eIx - sIx + 1; // Content count including possible separators + int sepCnt = sLen > 76 ? (sArr[76] == '\r' ? cCnt / 78 : 0) << 1 : 0; + + int len = ((cCnt - sepCnt) * 6 >> 3) - pad; // The number of decoded bytes + byte[] dArr = new byte[len]; // Preallocate byte[] of exact length + + // Decode all but the last 0 - 2 bytes. + int d = 0; + for (int cc = 0, eLen = (len / 3) * 3; d < eLen;) { + // Assemble three bytes into an int from four "valid" characters. + int i = IA[sArr[sIx++]] << 18 | IA[sArr[sIx++]] << 12 | IA[sArr[sIx++]] << 6 | IA[sArr[sIx++]]; + + // Add the bytes + dArr[d++] = (byte) (i >> 16); + dArr[d++] = (byte) (i >> 8); + dArr[d++] = (byte) i; + + // If line separator, jump over it. + if (sepCnt > 0 && ++cc == 19) { + sIx += 2; + cc = 0; + } + } + + if (d < len) { + // Decode last 1-3 bytes (incl '=') into 1-3 bytes + int i = 0; + for (int j = 0; sIx <= eIx - pad; j++) + i |= IA[sArr[sIx++]] << (18 - j * 6); + + for (int r = 16; d < len; r -= 8) + dArr[d++] = (byte) (i >> r); + } + + return dArr; + } + + // **************************************************************************************** + // * String version + // **************************************************************************************** + + /** Encodes a raw byte array into a BASE64 String representation i accordance with RFC 2045. + * @param sArr The bytes to convert. If null or length 0 an empty array will be returned. + * @param lineSep Optional "\r\n" after 76 characters, unless end of file.
+ * No line separator will be in breach of RFC 2045 which specifies max 76 per line but will be a + * little faster. + * @return A BASE64 encoded array. Never null. + */ + public final static String encodeToString(byte[] sArr, boolean lineSep) + { + // Reuse char[] since we can't create a String incrementally anyway and StringBuffer/Builder would be slower. + return new String(encodeToChar(sArr, lineSep)); + } + + /** Decodes a BASE64 encoded String. All illegal characters will be ignored and can handle both strings with + * and without line separators.
+ * Note! It can be up to about 2x the speed to call decode(str.toCharArray()) instead. That + * will create a temporary array though. This version will use str.charAt(i) to iterate the string. + * @param str The source string. null or length 0 will return an empty array. + * @return The decoded array of bytes. May be of length 0. Will be null if the legal characters + * (including '=') isn't divideable by 4. (I.e. definitely corrupted). + */ + public final static byte[] decode(String str) + { + // Check special case + int sLen = str != null ? str.length() : 0; + if (sLen == 0) + return new byte[0]; + + // Count illegal characters (including '\r', '\n') to know what size the returned array will be, + // so we don't have to reallocate & copy it later. + int sepCnt = 0; // Number of separator characters. (Actually illegal characters, but that's a bonus...) + for (int i = 0; i < sLen; i++) // If input is "pure" (I.e. no line separators or illegal chars) base64 this loop can be commented out. + if (IA[str.charAt(i)] < 0) + sepCnt++; + + // Check so that legal chars (including '=') are evenly divideable by 4 as specified in RFC 2045. + if ((sLen - sepCnt) % 4 != 0) + return null; + + // Count '=' at end + int pad = 0; + for (int i = sLen; i > 1 && IA[str.charAt(--i)] <= 0;) + if (str.charAt(i) == '=') + pad++; + + int len = ((sLen - sepCnt) * 6 >> 3) - pad; + + byte[] dArr = new byte[len]; // Preallocate byte[] of exact length + + for (int s = 0, d = 0; d < len;) { + // Assemble three bytes into an int from four "valid" characters. + int i = 0; + for (int j = 0; j < 4; j++) { // j only increased if a valid char was found. + int c = IA[str.charAt(s++)]; + if (c >= 0) + i |= c << (18 - j * 6); + else + j--; + } + // Add the bytes + dArr[d++] = (byte) (i >> 16); + if (d < len) { + dArr[d++]= (byte) (i >> 8); + if (d < len) + dArr[d++] = (byte) i; + } + } + return dArr; + } + + /** Decodes a BASE64 encoded string that is known to be resonably well formatted. The method is about twice as + * fast as {@link #decode(String)}. The preconditions are:
+ * + The array must have a line length of 76 chars OR no line separators at all (one line).
+ * + Line separator must be "\r\n", as specified in RFC 2045 + * + The array must not contain illegal characters within the encoded string
+ * + The array CAN have illegal characters at the beginning and end, those will be dealt with appropriately.
+ * @param s The source string. Length 0 will return an empty array. null will throw an exception. + * @return The decoded array of bytes. May be of length 0. + */ + public final static byte[] decodeFast(String s) + { + // Check special case + int sLen = s.length(); + if (sLen == 0) + return new byte[0]; + + int sIx = 0, eIx = sLen - 1; // Start and end index after trimming. + + // Trim illegal chars from start + while (sIx < eIx && IA[s.charAt(sIx) & 0xff] < 0) + sIx++; + + // Trim illegal chars from end + while (eIx > 0 && IA[s.charAt(eIx) & 0xff] < 0) + eIx--; + + // get the padding count (=) (0, 1 or 2) + int pad = s.charAt(eIx) == '=' ? (s.charAt(eIx - 1) == '=' ? 2 : 1) : 0; // Count '=' at end. + int cCnt = eIx - sIx + 1; // Content count including possible separators + int sepCnt = sLen > 76 ? (s.charAt(76) == '\r' ? cCnt / 78 : 0) << 1 : 0; + + int len = ((cCnt - sepCnt) * 6 >> 3) - pad; // The number of decoded bytes + byte[] dArr = new byte[len]; // Preallocate byte[] of exact length + + // Decode all but the last 0 - 2 bytes. + int d = 0; + for (int cc = 0, eLen = (len / 3) * 3; d < eLen;) { + // Assemble three bytes into an int from four "valid" characters. + int i = IA[s.charAt(sIx++)] << 18 | IA[s.charAt(sIx++)] << 12 | IA[s.charAt(sIx++)] << 6 | IA[s.charAt(sIx++)]; + + // Add the bytes + dArr[d++] = (byte) (i >> 16); + dArr[d++] = (byte) (i >> 8); + dArr[d++] = (byte) i; + + // If line separator, jump over it. + if (sepCnt > 0 && ++cc == 19) { + sIx += 2; + cc = 0; + } + } + + if (d < len) { + // Decode last 1-3 bytes (incl '=') into 1-3 bytes + int i = 0; + for (int j = 0; sIx <= eIx - pad; j++) + i |= IA[s.charAt(sIx++)] << (18 - j * 6); + + for (int r = 16; d < len; r -= 8) + dArr[d++] = (byte) (i >> r); + } + + return dArr; + } +} \ No newline at end of file diff --git a/src/main/net/tuis/resolver/CachedHTTPRepository.java b/src/main/net/tuis/resolver/CachedHTTPRepository.java index 41206ee..a79dfd4 100644 --- a/src/main/net/tuis/resolver/CachedHTTPRepository.java +++ b/src/main/net/tuis/resolver/CachedHTTPRepository.java @@ -170,7 +170,6 @@ public Thread newThread(Runnable r) { private static final String MD5 = "CURL_MD5"; private static final String CHARSET = "CURL_CHARSET"; private static final String MODIFIED = "CURL_MODIFIED"; - private static final String MODIFIEDHR = "CURL_MODIFIEDHR"; private static final String RESKEY = "CURL_RESKEY"; private static final String TOUCHTIME = "CURL_TOUCHED"; private static final String SPEEDTIME = "CURL_SPEED"; @@ -259,11 +258,7 @@ private static final long requireLongProperty(final Properties props, final Stri * @return the string representation of the bytes. */ private static final String md5ToString(final byte[] md5) { - final StringBuilder sb = new StringBuilder(16); - for (byte b : md5) { - sb.append(String.format("%02x", b)); - } - return sb.toString(); + return Base64.encodeToString(md5, false); } /** @@ -379,7 +374,8 @@ private static String getCacheLocation(URL url) { // needs to be a stringBuffer to use appendReplacement final StringBuffer sb = new StringBuffer(); sb.append(url.getProtocol()).append("/"); - sb.append(url.getAuthority()); + final String auth = url.getAuthority().replace(':', '~'); + sb.append(auth); final String path = url.getPath(); sb.append(path); final String query = url.getQuery(); @@ -524,6 +520,17 @@ public void run() { } } + private static final String getFirstHeader(final Map> headers, final String key) { + final List lst = headers.get(key); + if (lst == null) { + return null; + } + if (!lst.isEmpty()) { + return lst.get(0); + } + return null; + } + // Where the cache is. private final File cachefolder; @@ -829,7 +836,7 @@ public Resource resolve(final String publicID, final URL url) throws IOException // when do the properties say we expire. final long expires = requireLongProperty(props, EXPIRES); - final long lastmod = requireLongProperty(props, MODIFIED); + final String lastmod = requireProperty(props, MODIFIED); final String xmd5 = requireProperty(props, MD5); final String charset = requireProperty(props, CHARSET); @@ -947,7 +954,7 @@ public Resource resolve(final String publicID, final URL url) throws IOException * @throws IOException for the normal reasons. */ private HttpURLConnection getConnection(final String method, final URL url, - final long lastmod) throws IOException { + final String lastmod) throws IOException { final HttpURLConnection con = (HttpURLConnection) url.openConnection(); // we are a cache ourselves... we want to punch through to the real source. con.setUseCaches(false); @@ -956,9 +963,10 @@ private HttpURLConnection getConnection(final String method, final URL url, // we only get the HEAD if we previously had the file, but it is out of // date... con.setRequestMethod(method); - if (lastmod > 0L) { + if (lastmod != null && lastmod.length() > 0) { // we expect there to be a previous copy, with this mod time. - con.setIfModifiedSince(lastmod); + // avoid bug with getLastModified not parsing time-zone.... + con.addRequestProperty("If-Modified-Since", lastmod); } // set our user-agent. con.setRequestProperty("User-Agent", useragent.get()); @@ -979,7 +987,7 @@ private HttpURLConnection getConnection(final String method, final URL url, * @throws CacheEntryCorruptException If the web resource has been changed. */ private void checkResource(final long time, final String lk, - final FileChannel channel, final long lastmod, final Properties props, + final FileChannel channel, final String lastmod, final Properties props, final URL url) throws IOException, CacheEntryCorruptException { // we need to check that the resource is up to date. // we use the 'HEAD' method to get the state @@ -992,11 +1000,12 @@ private void checkResource(final long time, final String lk, } final Map> headers = con.getHeaderFields(); final long exp = calcExpire(time, con.getExpiration(), headers.get("Cache-Control")); - final long lmod = con.getLastModified(); + // bug in getLastModified - does not parse the TimeZone... assumes UTC. + //final long lmod = con.getLastModified(); + final String lmod = getFirstHeader(headers, "Last-Modified"); props.setProperty(EXPIRES, Long.toString(exp)); props.setProperty(EXPIRESHR, String.format("%tc", exp)); - props.setProperty(MODIFIED, Long.toString(lmod)); - props.setProperty(MODIFIEDHR, String.format("%tc", lmod)); + props.setProperty(MODIFIED, lmod == null ? "" : lmod); // write out the properties propertiesToChannel(channel, props); return; @@ -1051,7 +1060,7 @@ private Resource recreateResource(final String lk, final String key, final FileC final File datafile, final URL url, final String publicID, final long time) throws IOException { // we need to update the resource. long start = System.currentTimeMillis(); - final HttpURLConnection con = getConnection("GET", url, 0L); + final HttpURLConnection con = getConnection("GET", url, null); try { final int stat = con.getResponseCode(); logHeaders(lk, con); @@ -1066,7 +1075,7 @@ private Resource recreateResource(final String lk, final String key, final FileC } final Map> headers = con.getHeaderFields(); final long exp = calcExpire(time, con.getExpiration(), headers.get("Cache-Control")); - final long lmod = con.getLastModified(); + final String lmod = getFirstHeader(headers, "Last-Modified"); final String enc = getCharset(con.getContentType()); final String servermd5 = con.getHeaderField("Content-MD5"); @@ -1141,8 +1150,7 @@ private Resource recreateResource(final String lk, final String key, final FileC props.setProperty(MD5, mymd5); props.setProperty(EXPIRES, Long.toString(exp)); props.setProperty(EXPIRESHR, String.format("%tc", exp)); - props.setProperty(MODIFIED, Long.toString(lmod)); - props.setProperty(MODIFIEDHR, String.format("%tc", lmod)); + props.setProperty(MODIFIED, lmod); props.setProperty(CHARSET, enc == null ? "" : enc); props.setProperty(SYSTEMURL, url.toExternalForm()); props.setProperty(RESKEY, key);