From ebaa5fb9651a72b4128235f97b4b7163a6593220 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Fri, 26 Jul 2019 11:19:52 +0800 Subject: [PATCH 01/31] add threads and retry --- library/library.iml | 15 +- .../com/qiniu/android/ResumeUploadTest.java | 6 +- .../com/qiniu/android/common/AutoZone.java | 14 +- .../com/qiniu/android/common/Constants.java | 2 +- .../qiniu/android/storage/Configuration.java | 8 +- .../android/storage/ResumeUploaderFast.java | 566 ++++++++++++++++++ .../qiniu/android/storage/UploadManager.java | 42 +- .../com/qiniu/android/utils/StringUtils.java | 23 + 8 files changed, 651 insertions(+), 25 deletions(-) create mode 100644 library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java diff --git a/library/library.iml b/library/library.iml index ac6064470..e03a36633 100644 --- a/library/library.iml +++ b/library/library.iml @@ -75,7 +75,8 @@ - + + @@ -84,21 +85,17 @@ + + - - - - - - - - + + \ No newline at end of file diff --git a/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java b/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java index b78cf60c4..6da6c0604 100644 --- a/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java +++ b/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java @@ -83,7 +83,7 @@ public void progress(String key, double percent) { public void setUp() throws Exception { Configuration config = new Configuration.Builder().build(); - uploadManager = new UploadManager(config); + uploadManager = new UploadManager(config,3); ACollectUploadInfoTest.testInit(); } @@ -189,8 +189,8 @@ public void test4M1k2() throws Throwable { } @LargeTest - public void test4M() throws Throwable { - template(1024 * 4); + public void testResumeUploadFast() throws Throwable { + template(1024 * 98); } // @LargeTest diff --git a/library/src/main/java/com/qiniu/android/common/AutoZone.java b/library/src/main/java/com/qiniu/android/common/AutoZone.java index ab0f120dc..decb79ae6 100644 --- a/library/src/main/java/com/qiniu/android/common/AutoZone.java +++ b/library/src/main/java/com/qiniu/android/common/AutoZone.java @@ -21,17 +21,23 @@ public final class AutoZone extends Zone { /** * 自动判断机房 */ - public static final AutoZone autoZone = new AutoZone(); private final String ucServer; private Map zones = new ConcurrentHashMap<>(); private Client client = new Client(); + /** + * default useHttps to req autoZone + */ public AutoZone() { - this("https://uc.qbox.me"); + this(true); } - AutoZone(String ucServer) { - this.ucServer = ucServer; + public AutoZone(boolean useHttps){ + if(useHttps){ + this.ucServer = "https://uc.qbox.me"; + }else{ + this.ucServer = "http://uc.qbox.me"; + } } private void getZoneJsonAsync(ZoneIndex index, CompletionHandler handler) { diff --git a/library/src/main/java/com/qiniu/android/common/Constants.java b/library/src/main/java/com/qiniu/android/common/Constants.java index 55ec083dd..b66b2aec1 100644 --- a/library/src/main/java/com/qiniu/android/common/Constants.java +++ b/library/src/main/java/com/qiniu/android/common/Constants.java @@ -2,7 +2,7 @@ public final class Constants { - public static final String VERSION = "7.3.15"; + public static final String VERSION = "7.3.17"; public static final String UTF_8 = "utf-8"; } diff --git a/library/src/main/java/com/qiniu/android/storage/Configuration.java b/library/src/main/java/com/qiniu/android/storage/Configuration.java index 0e8e3734b..ec5febc46 100644 --- a/library/src/main/java/com/qiniu/android/storage/Configuration.java +++ b/library/src/main/java/com/qiniu/android/storage/Configuration.java @@ -83,8 +83,14 @@ private Configuration(Builder builder) { proxy = builder.proxy; urlConverter = builder.urlConverter; + AutoZone aotuZone=null; + if(builder.useHttps){ + aotuZone= new AutoZone(); + }else { + aotuZone= new AutoZone(false); + } - zone = builder.zone == null ? AutoZone.autoZone : builder.zone; + zone = builder.zone == null ? aotuZone : builder.zone; dns = builder.dns; } diff --git a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java new file mode 100644 index 000000000..1b549be3c --- /dev/null +++ b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java @@ -0,0 +1,566 @@ +package com.qiniu.android.storage; + +import com.qiniu.android.http.Client; +import com.qiniu.android.http.CompletionHandler; +import com.qiniu.android.http.ProgressHandler; +import com.qiniu.android.http.ResponseInfo; +import com.qiniu.android.utils.AndroidNetwork; +import com.qiniu.android.utils.Crc32; +import com.qiniu.android.utils.StringMap; +import com.qiniu.android.utils.StringUtils; +import com.qiniu.android.utils.UrlSafeBase64; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; + +import static java.lang.String.format; + +/** + * Created by jemy on 2019/7/9. + */ + +/** + * 线程不安全：contexts，crc32，getBlockInfo() + * 需要它不安全的有：upHost + * progress回调不能根据offset，要根据已经上传的总大小 + * 修正了preQurey err + * update AutoZone construct to query zone use http or https + */ +public class ResumeUploaderFast implements Runnable { + + /** + * 文件总大小 + */ + private final long totalSize; + + private final String key; + + private final UpCompletionHandler completionHandler; + + private final UploadOptions options; + + private final Client client; + + private final Configuration config; + /** + * 保存每一块上传返回的ctx + */ + private final String[] contexts; + /** + * headers + */ + private final StringMap headers; + /** + * 修改时间 + */ + private final long modifyTime; + /** + * 断点续传记录文件 + */ + private final String recorderKey; + /** + * 文件对象 + */ + private RandomAccessFile file; + /** + * 待上传文件 + */ + private File f; + /** + * 上传token + */ + private UpToken token; + /** + * 分块数据 + */ + private Map blockInfo; + /** + * 上传域名 + */ + private String upHost; + /** + * 总块数 + */ + private int tblock; + /** + * 重传域名数 + */ + private int retried = 0; + /** + * 单域名检测次数 + */ + private int singleDomainRetry = 1; + /** + * 线程数量 + */ + private int multithread; + /** + * 第一个任务 + */ + private boolean firsttask = true; + /** + * 每块偏移位子 + * use 断点续传 + */ + private Long[] offsets; + private int h = 0; + /** + * 重传过程，确认能成功上传的host之后，update checkDomainSuccess status + */ + private boolean checkDomainSuccess = false; + private int upRetry = 0; + + ResumeUploaderFast(Client client, Configuration config, File f, String key, UpToken token, + final UpCompletionHandler completionHandler, UploadOptions options, String recorderKey, int multithread) { + this.client = client; + this.config = config; + this.f = f; + this.recorderKey = recorderKey; + this.totalSize = f.length(); + this.key = key; + this.headers = new StringMap().put("Authorization", "UpToken " + token.token); + this.file = null; + this.multithread = multithread; + this.completionHandler = new UpCompletionHandler() { + @Override + public void complete(String key, ResponseInfo info, JSONObject response) { + if (file != null) { + try { + file.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + completionHandler.complete(key, info, response); + } + }; + this.options = options != null ? options : UploadOptions.defaultOptions(); + int totalBlock = (int) totalSize / Configuration.BLOCK_SIZE; + tblock = ((totalSize % Configuration.BLOCK_SIZE) == 0) ? totalBlock : totalBlock + 1; + this.offsets = new Long[tblock]; + contexts = new String[tblock]; + modifyTime = f.lastModified(); + this.token = token; + this.blockInfo = new LinkedHashMap<>(); + } + + @Override + public void run() { + try { + file = new RandomAccessFile(f, "r"); + } catch (FileNotFoundException e) { + e.printStackTrace(); + completionHandler.complete(key, ResponseInfo.fileError(e, token), null); + return; + } + putBlockInfo(); + upHost = config.zone.upHost(token.token, config.useHttps, null); + BlockElement fblock = getBlockInfo(); + mkblk(fblock.getOffset(), fblock.getBlocksize(), upHost); + } + + + /** + * cut file to blockInfo + */ + private void putBlockInfo() { + Long[] offsets = recoveryFromRecord(); + int lastBlock = tblock - 1; + if (offsets == null) { + for (int i = 0; i < lastBlock; i++) { + blockInfo.put((long) i * Configuration.BLOCK_SIZE, Configuration.BLOCK_SIZE); + } + blockInfo.put((long) lastBlock * Configuration.BLOCK_SIZE, (int) (totalSize - lastBlock * Configuration.BLOCK_SIZE)); + } else {//有断点文件 + HashSet set = new HashSet(Arrays.asList(offsets)); + for (int i = 0; i < lastBlock; i++) { + Long offset = (long) i * Configuration.BLOCK_SIZE; + if (!set.contains(offset)) { + blockInfo.put(offset, Configuration.BLOCK_SIZE); + } else { + offsets[i] = offset; + h += 1; + } + } + Long offset = (long) lastBlock * Configuration.BLOCK_SIZE; + if (!set.contains(offset)) { + blockInfo.put(offset, (int) (totalSize - lastBlock * Configuration.BLOCK_SIZE)); + } else { + offsets[lastBlock] = offset; + h += 1; + } + } + + } + + /** + * get next block use to upload + * + * @return + */ + private synchronized BlockElement getBlockInfo() { + Iterator> it = blockInfo.entrySet().iterator(); + long offset = 0; + int blockSize = 0; + while (it.hasNext()) { + Map.Entry entry = it.next(); + offset = entry.getKey(); + blockSize = entry.getValue(); + blockInfo.remove(offset); + break; + } + return new BlockElement(offset, blockSize); + } + + class UploadTread extends Thread { + private long offset; + private int blockSize; + private String upHost; + + UploadTread(long offset, int blockSize, String upHost) { + this.offset = offset; + this.blockSize = blockSize; + this.upHost = upHost; + } + + @Override + public void run() { + super.run(); + mkblk(offset, blockSize, upHost); + } + } + + /** + * 创建块，并上传内容 + * + * @param upHost 上传主机 + * @param offset 本地文件偏移量 + * @param blockSize 分块的块大小 + */ + private void mkblk(long offset, int blockSize, String upHost) { + String path = format(Locale.ENGLISH, "/mkblk/%d", blockSize); + byte[] chunkBuffer = new byte[blockSize]; + try { + file.seek(offset); + file.read(chunkBuffer, 0, blockSize); + } catch (IOException e) { + completionHandler.complete(key, ResponseInfo.fileError(e, token), null); + return; + } + + long crc32 = Crc32.bytes(chunkBuffer, 0, blockSize); + String postUrl = String.format("%s%s", upHost, path); + + post(postUrl, chunkBuffer, 0, blockSize, getProgressHandler(), + getCompletionHandler(offset, blockSize, crc32), options.cancellationSignal); + } + + private void makeFile(String upHost, CompletionHandler _completionHandler, UpCancellationSignal c) { + String mime = format(Locale.ENGLISH, "/mimeType/%s/fname/%s", + UrlSafeBase64.encodeToString(options.mimeType), UrlSafeBase64.encodeToString(f.getName())); + + String keyStr = ""; + if (key != null) { + keyStr = format("/key/%s", UrlSafeBase64.encodeToString(key)); + } + + String paramStr = ""; + if (options.params.size() != 0) { + String str[] = new String[options.params.size()]; + int j = 0; + for (Map.Entry i : options.params.entrySet()) { + str[j++] = format(Locale.ENGLISH, "%s/%s", i.getKey(), UrlSafeBase64.encodeToString(i.getValue())); + } + paramStr = "/" + StringUtils.join(str, "/"); + } + String path = format(Locale.ENGLISH, "/mkfile/%d%s%s%s", totalSize, mime, keyStr, paramStr); + String bodyStr = StringUtils.join(contexts, ","); + byte[] data = bodyStr.getBytes(); + String postUrl = String.format("%s%s", upHost, path); + post(postUrl, data, 0, data.length, null, _completionHandler, c); + } + + private void post(String upHost, byte[] data, int offset, int dataSize, ProgressHandler progress, + CompletionHandler completion, UpCancellationSignal c) { + client.asyncPost(upHost, data, offset, dataSize, headers, token, totalSize, progress, completion, c); + } + + private ProgressHandler getProgressHandler() { + return new ProgressHandler() { + @Override + public void onProgress(long bytesWritten, long totalSize) { + long size = 0; + for (int i = 0; i < offsets.length; i++) { + if (offsets[i] != null && offsets[i] > 0) { + size += 1; + } + } + double percent = (double) size * Configuration.BLOCK_SIZE / totalSize; + if (percent > 0.95) { + percent = 0.95; + } + options.progressHandler.progress(key, percent); + } + }; + + } + + private CompletionHandler getMkfileCompletionHandler() { + return new CompletionHandler() { + @Override + public void complete(ResponseInfo info, JSONObject response) { + if (info.isNetworkBroken() && !AndroidNetwork.isNetWorkReady()) { + options.netReadyHandler.waitReady(); + if (!AndroidNetwork.isNetWorkReady()) { + completionHandler.complete(key, info, response); + return; + } + } + + if (info.isOK()) { + removeRecord(); + options.progressHandler.progress(key, 1.0); + completionHandler.complete(key, info, response); + return; + } + + // mkfile ，允许多重试一次 + if (info.needRetry() && retried < config.retryMax + 1) { + makeFile(upHost, getMkfileCompletionHandler(), options.cancellationSignal); + } + completionHandler.complete(key, info, response); + } + }; + } + + private CompletionHandler getCompletionHandler(final long offset, final int blockSize, final long crc32) { + return new CompletionHandler() { + @Override + public void complete(ResponseInfo info, JSONObject response) { + //网络断开或者无状态，检查3s，直接返回 + if (info.isNetworkBroken() && !AndroidNetwork.isNetWorkReady()) { + options.netReadyHandler.waitReady(); + if (!AndroidNetwork.isNetWorkReady()) { + completionHandler.complete(key, info, response); + return; + } + } + + //取消 + if (info.isCancelled()) { + completionHandler.complete(key, info, response); + return; + } + + //上传失败：重试 + //单个域名检测，重试3次 + //下一个域名进行重试 + //701 ctx不正确或者已经过期 + if (!isChunkOK(info, response)) { + if ((info.statusCode == 701 && checkRetried()) || + ((isNotChunkToQiniu(info, response) || info.needRetry()) && checkRetried())) {//三次，一个域名 + mkblk(offset, blockSize, upHost); + return; + } + + if (upHost != null + && ((isNotChunkToQiniu(info, response) || info.needRetry()) + && checkRetried())) { + mkblk(offset, blockSize, upHost); + return; + } + completionHandler.complete(key, info, response); + return; + } + + + //上传成功(伪成功)：检查response + String context = null; + + if (response == null && retried < config.retryMax) { + mkblk(offset, blockSize, upHost); + return; + } + long crc = 0; + Exception tempE = null; + try { + context = response.getString("ctx"); + crc = response.getLong("crc32"); + } catch (Exception e) { + tempE = e; + e.printStackTrace(); + } + if ((context == null || crc != crc32) && retried < config.retryMax) { + mkblk(offset, blockSize, upHost); + return; + } + if (context == null) { + String error = "get context failed."; + if (tempE != null) { + error += "\n"; + error += tempE.getMessage(); + } + ResponseInfo info2 = ResponseInfo.errorInfo(info, ResponseInfo.UnknownError, error); + completionHandler.complete(key, info2, response); + return; + } + if (crc != crc32) { + String error = "block's crc32 is not match. local: " + crc32 + ", remote: " + crc; + ResponseInfo info2 = ResponseInfo.errorInfo(info, ResponseInfo.Crc32NotMatch, error); + completionHandler.complete(key, info2, response); + return; + } + + synchronized (this) { + contexts[(int) (offset / Configuration.BLOCK_SIZE)] = context; + offsets[(int) (offset / Configuration.BLOCK_SIZE)] = offset; + record(offsets); + } + h += 1; + if (h >= tblock) { + makeFile(upHost, getMkfileCompletionHandler(), options.cancellationSignal); + } + + if (firsttask) { + if (blockInfo.size() < multithread) { + multithread = blockInfo.size(); + } + for (int i = 0; i < multithread; i++) { + BlockElement mblock = getBlockInfo(); + Thread t = new UploadTread(mblock.getOffset(), mblock.getBlocksize(), upHost); + t.start(); + } + firsttask = false; + } else { + BlockElement mblock = getBlockInfo(); + if (mblock.getOffset() != 0 && mblock.getBlocksize() != 0) + new UploadTread(mblock.getOffset(), mblock.getBlocksize(), upHost).start(); + } + + + } + }; + } + + private boolean checkRetried() { + boolean retry = singleDomainRetry < config.retryMax && retried < config.retryMax; + if (checkDomainSuccess && upRetry < config.retryMax) { + upRetry = +1; + return true; + } + if (retry) { + singleDomainRetry += 1; + return true; + } else if (retried < config.retryMax - 1) { + singleDomainRetry = 1; + retried += 1; + upHost = config.zone.upHost(token.token, config.useHttps, upHost); + return true; + } + return retry; + } + + private boolean isChunkOK(ResponseInfo info, JSONObject response) { + return info.statusCode == 200 && info.error == null && (info.hasReqId() || isChunkResOK(response)); + } + + private boolean isChunkResOK(JSONObject response) { + try { + // getXxxx 若获取不到值,会抛出异常 + response.getString("ctx"); + response.getLong("crc32"); + } catch (Exception e) { + return false; + } + return true; + } + + private boolean isNotChunkToQiniu(ResponseInfo info, JSONObject response) { + return info.statusCode < 500 && info.statusCode >= 200 && (!info.hasReqId() && !isChunkResOK(response)); + } + + private Long[] recoveryFromRecord() { + + if (config.recorder == null) { + return null; + } + byte[] data = config.recorder.get(recorderKey); + if (data == null) { + return null; + } + String jsonStr = new String(data); + JSONObject obj; + try { + obj = new JSONObject(jsonStr); + } catch (JSONException e) { + e.printStackTrace(); + return null; + } + JSONArray offsetsArray = obj.optJSONArray("offsets"); + long modify = obj.optLong("modify_time", 0); + long fSize = obj.optLong("size", 0); + JSONArray array = obj.optJSONArray("contexts"); + if (offsetsArray.length() == 0 || modify != modifyTime || fSize != totalSize || array == null || array.length() == 0) { + return null; + } + for (int i = 0; i < array.length(); i++) { + contexts[i] = array.optString(i); + } + for (int i = 0; i < array.length(); i++) { + String offset = offsetsArray.optString(i); + if (offset != null && !offset.equals("null")) { + offsets[i] = Long.parseLong(offset); + } + } + return offsets; + } + + private void removeRecord() { + if (config.recorder != null) { + config.recorder.del(recorderKey); + } + } + + + private void record(Long[] offsets) { + if (config.recorder == null || offsets.length == 0) { + return; + } + String data = format(Locale.ENGLISH, "{\"size\":%d,\"offsets\":[%s], \"modify_time\":%d, \"contexts\":[%s]}", + totalSize, StringUtils.jsonJoin(offsets), modifyTime, StringUtils.jsonJoin(contexts)); + config.recorder.set(recorderKey, data.getBytes()); + } + + + class BlockElement { + private long offset; + private int blocksize; + + BlockElement(long offset, int blocksize) { + this.offset = offset; + this.blocksize = blocksize; + } + + public long getOffset() { + return offset; + } + + public int getBlocksize() { + return blocksize; + } + } + + +} diff --git a/library/src/main/java/com/qiniu/android/storage/UploadManager.java b/library/src/main/java/com/qiniu/android/storage/UploadManager.java index 517b4c678..388db2c45 100644 --- a/library/src/main/java/com/qiniu/android/storage/UploadManager.java +++ b/library/src/main/java/com/qiniu/android/storage/UploadManager.java @@ -21,25 +21,48 @@ public final class UploadManager { private final Configuration config; private final Client client; + private int multitread = 0; + private static int DEF_THREAD_NUM=3; + /** + * default 3 Threads + */ public UploadManager() { - this(new Configuration.Builder().build()); + this(new Configuration.Builder().build(),DEF_THREAD_NUM); } + /** + * + * @param config Configuration, default 1 Threads + */ public UploadManager(Configuration config) { this.config = config; this.client = new Client(config.proxy, config.connectTimeout, config.responseTimeout, config.urlConverter, config.dns); } - public UploadManager(Recorder recorder, KeyGenerator keyGen) { - this(new Configuration.Builder().recorder(recorder, keyGen).build()); + public UploadManager(Configuration config, int multitread) { + this.config = config; + this.multitread = multitread>=0?multitread:DEF_THREAD_NUM; + this.client = new Client(config.proxy, config.connectTimeout, config.responseTimeout, + config.urlConverter, config.dns); } public UploadManager(Recorder recorder) { this(recorder, null); } + public UploadManager(Recorder recorder, KeyGenerator keyGen) { + this(new Configuration.Builder().recorder(recorder, keyGen).build()); + } + + public UploadManager(Recorder recorder,int multitread) { + this(recorder, null,multitread); + } + public UploadManager(Recorder recorder, KeyGenerator keyGen,int multitread) { + this(new Configuration.Builder().recorder(recorder, keyGen).build(),multitread); + } + private static boolean areInvalidArg(final String key, byte[] data, File f, String token, UpToken decodedToken, final UpCompletionHandler complete) { if (complete == null) { @@ -172,10 +195,15 @@ public void onSuccess() { } String recorderKey = config.keyGen.gen(key, file); final WarpHandler completionHandler = warpHandler(complete, file != null ? file.length() : 0); - ResumeUploader uploader = new ResumeUploader(client, config, file, key, - decodedToken, completionHandler, options, recorderKey); - - AsyncRun.runInMain(uploader); + if(multitread==0) { + ResumeUploader uploader = new ResumeUploader(client, config, file, key, + decodedToken, completionHandler, options, recorderKey); + AsyncRun.runInMain(uploader); + }else{ + ResumeUploaderFast uploader = new ResumeUploaderFast(client, config, file, key, + decodedToken, completionHandler, options, recorderKey,multitread); + AsyncRun.runInMain(uploader); + } } @Override diff --git a/library/src/main/java/com/qiniu/android/utils/StringUtils.java b/library/src/main/java/com/qiniu/android/utils/StringUtils.java index 2b4f57ba7..a13d572a0 100644 --- a/library/src/main/java/com/qiniu/android/utils/StringUtils.java +++ b/library/src/main/java/com/qiniu/android/utils/StringUtils.java @@ -51,6 +51,7 @@ public static String join(String[] array, String sep) { return buf.toString(); } + /** * 以json元素的方式连接字符串中元素 *

@@ -83,6 +84,28 @@ public static String jsonJoin(String[] array) { return buf.toString(); } + + public static String jsonJoin(Long[] array) { + return jsonJoin(longToString(array)); + } + + public static String[] longToString(Long longArray[]) { + if (longArray == null || longArray.length < 1) { + return null; + } + String stringArray[] = new String[longArray.length]; + for (int i = 0; i < stringArray.length; i++) { + try { + stringArray[i] = String.valueOf(longArray[i]); + } catch (NumberFormatException e) { + stringArray[i] = null; + continue; + } + } + return stringArray; + + } + public static byte[] utf8Bytes(String data) { try { return data.getBytes(Constants.UTF_8); From 9e8eaff364621b30fd869515618f1e5f8abb097f Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Fri, 26 Jul 2019 16:46:04 +0800 Subject: [PATCH 02/31] multithreads_retry --- library/library.iml | 4 ++++ .../com/qiniu/android/ResumeUploadTest.java | 2 +- .../com/qiniu/android/common/AutoZone.java | 6 ++--- .../qiniu/android/storage/Configuration.java | 12 +++++----- .../qiniu/android/storage/UploadManager.java | 24 +++++++++---------- 5 files changed, 26 insertions(+), 22 deletions(-) diff --git a/library/library.iml b/library/library.iml index e03a36633..0035a0e47 100644 --- a/library/library.iml +++ b/library/library.iml @@ -75,6 +75,7 @@ + @@ -90,7 +91,10 @@ + + + diff --git a/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java b/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java index 6da6c0604..07ef4bec5 100644 --- a/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java +++ b/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java @@ -83,7 +83,7 @@ public void progress(String key, double percent) { public void setUp() throws Exception { Configuration config = new Configuration.Builder().build(); - uploadManager = new UploadManager(config,3); + uploadManager = new UploadManager(config, 3); ACollectUploadInfoTest.testInit(); } diff --git a/library/src/main/java/com/qiniu/android/common/AutoZone.java b/library/src/main/java/com/qiniu/android/common/AutoZone.java index decb79ae6..102bb72d0 100644 --- a/library/src/main/java/com/qiniu/android/common/AutoZone.java +++ b/library/src/main/java/com/qiniu/android/common/AutoZone.java @@ -32,10 +32,10 @@ public AutoZone() { this(true); } - public AutoZone(boolean useHttps){ - if(useHttps){ + public AutoZone(boolean useHttps) { + if (useHttps) { this.ucServer = "https://uc.qbox.me"; - }else{ + } else { this.ucServer = "http://uc.qbox.me"; } } diff --git a/library/src/main/java/com/qiniu/android/storage/Configuration.java b/library/src/main/java/com/qiniu/android/storage/Configuration.java index ec5febc46..bcf9cd7f3 100644 --- a/library/src/main/java/com/qiniu/android/storage/Configuration.java +++ b/library/src/main/java/com/qiniu/android/storage/Configuration.java @@ -83,14 +83,14 @@ private Configuration(Builder builder) { proxy = builder.proxy; urlConverter = builder.urlConverter; - AutoZone aotuZone=null; - if(builder.useHttps){ - aotuZone= new AutoZone(); - }else { - aotuZone= new AutoZone(false); + AutoZone autoZone = null; + if (builder.useHttps) { + autoZone = new AutoZone(); + } else { + autoZone = new AutoZone(false); } - zone = builder.zone == null ? aotuZone : builder.zone; + zone = builder.zone == null ? autoZone : builder.zone; dns = builder.dns; } diff --git a/library/src/main/java/com/qiniu/android/storage/UploadManager.java b/library/src/main/java/com/qiniu/android/storage/UploadManager.java index 388db2c45..3385f832d 100644 --- a/library/src/main/java/com/qiniu/android/storage/UploadManager.java +++ b/library/src/main/java/com/qiniu/android/storage/UploadManager.java @@ -21,18 +21,17 @@ public final class UploadManager { private final Configuration config; private final Client client; - private int multitread = 0; - private static int DEF_THREAD_NUM=3; + private int multithread = 0; + private static int DEF_THREAD_NUM = 3; /** * default 3 Threads */ public UploadManager() { - this(new Configuration.Builder().build(),DEF_THREAD_NUM); + this(new Configuration.Builder().build(), DEF_THREAD_NUM); } /** - * * @param config Configuration, default 1 Threads */ public UploadManager(Configuration config) { @@ -43,7 +42,7 @@ public UploadManager(Configuration config) { public UploadManager(Configuration config, int multitread) { this.config = config; - this.multitread = multitread>=0?multitread:DEF_THREAD_NUM; + this.multithread = multitread >= 0 ? multitread : DEF_THREAD_NUM; this.client = new Client(config.proxy, config.connectTimeout, config.responseTimeout, config.urlConverter, config.dns); } @@ -56,11 +55,12 @@ public UploadManager(Recorder recorder, KeyGenerator keyGen) { this(new Configuration.Builder().recorder(recorder, keyGen).build()); } - public UploadManager(Recorder recorder,int multitread) { - this(recorder, null,multitread); + public UploadManager(Recorder recorder, int multitread) { + this(recorder, null, multitread); } - public UploadManager(Recorder recorder, KeyGenerator keyGen,int multitread) { - this(new Configuration.Builder().recorder(recorder, keyGen).build(),multitread); + + public UploadManager(Recorder recorder, KeyGenerator keyGen, int multitread) { + this(new Configuration.Builder().recorder(recorder, keyGen).build(), multitread); } private static boolean areInvalidArg(final String key, byte[] data, File f, String token, @@ -195,13 +195,13 @@ public void onSuccess() { } String recorderKey = config.keyGen.gen(key, file); final WarpHandler completionHandler = warpHandler(complete, file != null ? file.length() : 0); - if(multitread==0) { + if (multithread == 0) { ResumeUploader uploader = new ResumeUploader(client, config, file, key, decodedToken, completionHandler, options, recorderKey); AsyncRun.runInMain(uploader); - }else{ + } else { ResumeUploaderFast uploader = new ResumeUploaderFast(client, config, file, key, - decodedToken, completionHandler, options, recorderKey,multitread); + decodedToken, completionHandler, options, recorderKey, multithread); AsyncRun.runInMain(uploader); } } From 0620a6c292526e2de451e04bb7b69e54dad18ba7 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Fri, 26 Jul 2019 17:29:48 +0800 Subject: [PATCH 03/31] recode retry --- .../java/com/qiniu/android/storage/ResumeUploaderFast.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java index 1b549be3c..55e1cab67 100644 --- a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java +++ b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java @@ -336,9 +336,11 @@ public void complete(ResponseInfo info, JSONObject response) { return; } - // mkfile ，允许多重试一次 + // mkfile ，允许多重试一次，这里不需要重试时，成功与否都complete回调给客户端 if (info.needRetry() && retried < config.retryMax + 1) { makeFile(upHost, getMkfileCompletionHandler(), options.cancellationSignal); + retried += 1; + return; } completionHandler.complete(key, info, response); } @@ -391,6 +393,7 @@ && checkRetried())) { if (response == null && retried < config.retryMax) { mkblk(offset, blockSize, upHost); + retried += 1; return; } long crc = 0; @@ -403,6 +406,7 @@ && checkRetried())) { e.printStackTrace(); } if ((context == null || crc != crc32) && retried < config.retryMax) { + retried += 1; mkblk(offset, blockSize, upHost); return; } From e301472a7ff06bf2c729630891d16e920ba5fcf5 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Mon, 29 Jul 2019 13:00:04 +0800 Subject: [PATCH 04/31] update retry --- library/library.iml | 3 + .../com/qiniu/android/ResumeUploadTest.java | 2 +- .../android/storage/ResumeUploaderFast.java | 63 ++++++++----------- .../qiniu/android/storage/UploadManager.java | 4 +- .../com/qiniu/android/utils/StringUtils.java | 1 - 5 files changed, 31 insertions(+), 42 deletions(-) diff --git a/library/library.iml b/library/library.iml index 0035a0e47..d715e051f 100644 --- a/library/library.iml +++ b/library/library.iml @@ -75,6 +75,7 @@ + @@ -83,9 +84,11 @@ + + diff --git a/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java b/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java index 07ef4bec5..418fff585 100644 --- a/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java +++ b/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java @@ -197,4 +197,4 @@ public void testResumeUploadFast() throws Throwable { // public void test8M1k() throws Throwable{ // template(1024*8+1); // } -} +} \ No newline at end of file diff --git a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java index 55e1cab67..247c9664e 100644 --- a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java +++ b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java @@ -109,18 +109,14 @@ public class ResumeUploaderFast implements Runnable { /** * 第一个任务 */ - private boolean firsttask = true; + private boolean isFirstTask = true; /** * 每块偏移位子 * use 断点续传 */ private Long[] offsets; private int h = 0; - /** - * 重传过程，确认能成功上传的host之后，update checkDomainSuccess status - */ - private boolean checkDomainSuccess = false; - private int upRetry = 0; + private final int domainRetry = 3; ResumeUploaderFast(Client client, Configuration config, File f, String key, UpToken token, final UpCompletionHandler completionHandler, UploadOptions options, String recorderKey, int multithread) { @@ -176,15 +172,16 @@ public void run() { * cut file to blockInfo */ private void putBlockInfo() { - Long[] offsets = recoveryFromRecord(); + Long[] offs = recoveryFromRecord(); + int lastBlock = tblock - 1; - if (offsets == null) { + if (offs == null) { for (int i = 0; i < lastBlock; i++) { blockInfo.put((long) i * Configuration.BLOCK_SIZE, Configuration.BLOCK_SIZE); } blockInfo.put((long) lastBlock * Configuration.BLOCK_SIZE, (int) (totalSize - lastBlock * Configuration.BLOCK_SIZE)); } else {//有断点文件 - HashSet set = new HashSet(Arrays.asList(offsets)); + HashSet set = new HashSet(Arrays.asList(offs)); for (int i = 0; i < lastBlock; i++) { Long offset = (long) i * Configuration.BLOCK_SIZE; if (!set.contains(offset)) { @@ -202,7 +199,6 @@ private void putBlockInfo() { h += 1; } } - } /** @@ -367,33 +363,30 @@ public void complete(ResponseInfo info, JSONObject response) { } //上传失败：重试 - //单个域名检测，重试3次 - //下一个域名进行重试 - //701 ctx不正确或者已经过期 + //701: ctx不正确或者已经过期 if (!isChunkOK(info, response)) { - if ((info.statusCode == 701 && checkRetried()) || - ((isNotChunkToQiniu(info, response) || info.needRetry()) && checkRetried())) {//三次，一个域名 + if (info.statusCode == 701 && checkRetried()) { mkblk(offset, blockSize, upHost); + updateRetried(); return; } - if (upHost != null && ((isNotChunkToQiniu(info, response) || info.needRetry()) && checkRetried())) { mkblk(offset, blockSize, upHost); + updateRetried(); return; } completionHandler.complete(key, info, response); return; } - //上传成功(伪成功)：检查response String context = null; - if (response == null && retried < config.retryMax) { + if (response == null && checkRetried()) { mkblk(offset, blockSize, upHost); - retried += 1; + updateRetried(); return; } long crc = 0; @@ -405,9 +398,10 @@ && checkRetried())) { tempE = e; e.printStackTrace(); } - if ((context == null || crc != crc32) && retried < config.retryMax) { + if ((context == null || crc != crc32) && checkRetried()) { retried += 1; mkblk(offset, blockSize, upHost); + updateRetried(); return; } if (context == null) { @@ -431,13 +425,14 @@ && checkRetried())) { contexts[(int) (offset / Configuration.BLOCK_SIZE)] = context; offsets[(int) (offset / Configuration.BLOCK_SIZE)] = offset; record(offsets); + h += 1; } - h += 1; if (h >= tblock) { makeFile(upHost, getMkfileCompletionHandler(), options.cancellationSignal); + return; } - if (firsttask) { + if (isFirstTask) { if (blockInfo.size() < multithread) { multithread = blockInfo.size(); } @@ -446,34 +441,28 @@ && checkRetried())) { Thread t = new UploadTread(mblock.getOffset(), mblock.getBlocksize(), upHost); t.start(); } - firsttask = false; + isFirstTask = false; } else { BlockElement mblock = getBlockInfo(); if (mblock.getOffset() != 0 && mblock.getBlocksize() != 0) new UploadTread(mblock.getOffset(), mblock.getBlocksize(), upHost).start(); } - - } }; } - private boolean checkRetried() { - boolean retry = singleDomainRetry < config.retryMax && retried < config.retryMax; - if (checkDomainSuccess && upRetry < config.retryMax) { - upRetry = +1; - return true; - } - if (retry) { + private synchronized void updateRetried() { + if (singleDomainRetry < config.retryMax) { singleDomainRetry += 1; - return true; - } else if (retried < config.retryMax - 1) { + } else if (retried < domainRetry) { singleDomainRetry = 1; retried += 1; upHost = config.zone.upHost(token.token, config.useHttps, upHost); - return true; } - return retry; + } + + private boolean checkRetried() { + return retried < domainRetry; } private boolean isChunkOK(ResponseInfo info, JSONObject response) { @@ -537,7 +526,6 @@ private void removeRecord() { } } - private void record(Long[] offsets) { if (config.recorder == null || offsets.length == 0) { return; @@ -547,7 +535,6 @@ private void record(Long[] offsets) { config.recorder.set(recorderKey, data.getBytes()); } - class BlockElement { private long offset; private int blocksize; diff --git a/library/src/main/java/com/qiniu/android/storage/UploadManager.java b/library/src/main/java/com/qiniu/android/storage/UploadManager.java index 3385f832d..0a4bb6eaf 100644 --- a/library/src/main/java/com/qiniu/android/storage/UploadManager.java +++ b/library/src/main/java/com/qiniu/android/storage/UploadManager.java @@ -32,7 +32,7 @@ public UploadManager() { } /** - * @param config Configuration, default 1 Threads + * @param config Configuration, default 1 Thread */ public UploadManager(Configuration config) { this.config = config; @@ -42,7 +42,7 @@ public UploadManager(Configuration config) { public UploadManager(Configuration config, int multitread) { this.config = config; - this.multithread = multitread >= 0 ? multitread : DEF_THREAD_NUM; + this.multithread = multitread >= 1 ? multitread : DEF_THREAD_NUM; this.client = new Client(config.proxy, config.connectTimeout, config.responseTimeout, config.urlConverter, config.dns); } diff --git a/library/src/main/java/com/qiniu/android/utils/StringUtils.java b/library/src/main/java/com/qiniu/android/utils/StringUtils.java index a13d572a0..9591bea3f 100644 --- a/library/src/main/java/com/qiniu/android/utils/StringUtils.java +++ b/library/src/main/java/com/qiniu/android/utils/StringUtils.java @@ -51,7 +51,6 @@ public static String join(String[] array, String sep) { return buf.toString(); } - /** * 以json元素的方式连接字符串中元素 *

From cb6c7716e58661d126d813ac63ca4f08731892fa Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Mon, 29 Jul 2019 13:07:59 +0800 Subject: [PATCH 05/31] updata multithreads --- .../java/com/qiniu/android/storage/UploadManager.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/storage/UploadManager.java b/library/src/main/java/com/qiniu/android/storage/UploadManager.java index 0a4bb6eaf..fe7e03770 100644 --- a/library/src/main/java/com/qiniu/android/storage/UploadManager.java +++ b/library/src/main/java/com/qiniu/android/storage/UploadManager.java @@ -21,7 +21,7 @@ public final class UploadManager { private final Configuration config; private final Client client; - private int multithread = 0; + private int multithreads = 0; private static int DEF_THREAD_NUM = 3; /** @@ -42,7 +42,7 @@ public UploadManager(Configuration config) { public UploadManager(Configuration config, int multitread) { this.config = config; - this.multithread = multitread >= 1 ? multitread : DEF_THREAD_NUM; + this.multithreads = multitread >= 1 ? multitread : DEF_THREAD_NUM; this.client = new Client(config.proxy, config.connectTimeout, config.responseTimeout, config.urlConverter, config.dns); } @@ -195,13 +195,13 @@ public void onSuccess() { } String recorderKey = config.keyGen.gen(key, file); final WarpHandler completionHandler = warpHandler(complete, file != null ? file.length() : 0); - if (multithread == 0) { + if (multithreads == 1) { ResumeUploader uploader = new ResumeUploader(client, config, file, key, decodedToken, completionHandler, options, recorderKey); AsyncRun.runInMain(uploader); } else { ResumeUploaderFast uploader = new ResumeUploaderFast(client, config, file, key, - decodedToken, completionHandler, options, recorderKey, multithread); + decodedToken, completionHandler, options, recorderKey, multithreads); AsyncRun.runInMain(uploader); } } From 1733b2331c1eb1a1ce033e9d1cbd97518cdd4d27 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Tue, 30 Jul 2019 16:09:25 +0800 Subject: [PATCH 06/31] format code --- .../android/storage/ResumeUploaderFast.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java index 247c9664e..8f65a3aee 100644 --- a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java +++ b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java @@ -115,7 +115,10 @@ public class ResumeUploaderFast implements Runnable { * use 断点续传 */ private Long[] offsets; - private int h = 0; + /** + * 已上传块 + */ + private int upBlock = 0; private final int domainRetry = 3; ResumeUploaderFast(Client client, Configuration config, File f, String key, UpToken token, @@ -143,8 +146,8 @@ public void complete(String key, ResponseInfo info, JSONObject response) { } }; this.options = options != null ? options : UploadOptions.defaultOptions(); - int totalBlock = (int) totalSize / Configuration.BLOCK_SIZE; - tblock = ((totalSize % Configuration.BLOCK_SIZE) == 0) ? totalBlock : totalBlock + 1; + tblock = (int) (totalSize + Configuration.BLOCK_SIZE - 1) / Configuration.BLOCK_SIZE; + ; this.offsets = new Long[tblock]; contexts = new String[tblock]; modifyTime = f.lastModified(); @@ -188,7 +191,7 @@ private void putBlockInfo() { blockInfo.put(offset, Configuration.BLOCK_SIZE); } else { offsets[i] = offset; - h += 1; + upBlock += 1; } } Long offset = (long) lastBlock * Configuration.BLOCK_SIZE; @@ -196,7 +199,7 @@ private void putBlockInfo() { blockInfo.put(offset, (int) (totalSize - lastBlock * Configuration.BLOCK_SIZE)); } else { offsets[lastBlock] = offset; - h += 1; + upBlock += 1; } } } @@ -210,12 +213,11 @@ private synchronized BlockElement getBlockInfo() { Iterator> it = blockInfo.entrySet().iterator(); long offset = 0; int blockSize = 0; - while (it.hasNext()) { + if (it.hasNext()) { Map.Entry entry = it.next(); offset = entry.getKey(); blockSize = entry.getValue(); blockInfo.remove(offset); - break; } return new BlockElement(offset, blockSize); } @@ -425,9 +427,9 @@ && checkRetried())) { contexts[(int) (offset / Configuration.BLOCK_SIZE)] = context; offsets[(int) (offset / Configuration.BLOCK_SIZE)] = offset; record(offsets); - h += 1; + upBlock += 1; } - if (h >= tblock) { + if (upBlock >= tblock) { makeFile(upHost, getMkfileCompletionHandler(), options.cancellationSignal); return; } From 15083c2c1893a5ca4af6842a74712a3248ea68ae Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Wed, 31 Jul 2019 14:45:25 +0800 Subject: [PATCH 07/31] update threads and fix a record bug --- .../com/qiniu/android/ResumeUploadTest.java | 4 +- .../android/storage/ResumeUploaderFast.java | 78 +++++++++++-------- .../com/qiniu/android/utils/StringUtils.java | 10 ++- 3 files changed, 56 insertions(+), 36 deletions(-) diff --git a/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java b/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java index 418fff585..57b71f859 100644 --- a/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java +++ b/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java @@ -9,10 +9,12 @@ import com.qiniu.android.common.Zone; import com.qiniu.android.http.ResponseInfo; import com.qiniu.android.storage.Configuration; +import com.qiniu.android.storage.Recorder; import com.qiniu.android.storage.UpCompletionHandler; import com.qiniu.android.storage.UpProgressHandler; import com.qiniu.android.storage.UploadManager; import com.qiniu.android.storage.UploadOptions; +import com.qiniu.android.storage.persistent.FileRecorder; import junit.framework.Assert; @@ -120,7 +122,7 @@ public void complete(String k, ResponseInfo rinfo, JSONObject response) { // Assert.assertNotNull(resp); // String hash = resp.getString("hash"); // Assert.assertEquals(hash, Etag.file(f)); - TempFile.remove(f); + //TempFile.remove(f); // Assert.assertTrue("进度有变化，不大可能一直相同。" + getProgress(), !isProgressAllSame()); Log.d(TAG, getProgress()); ACollectUploadInfoTest.recordFileTest(); diff --git a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java index 8f65a3aee..8ce843b09 100644 --- a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java +++ b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java @@ -1,5 +1,7 @@ package com.qiniu.android.storage; +import android.util.Log; + import com.qiniu.android.http.Client; import com.qiniu.android.http.CompletionHandler; import com.qiniu.android.http.ProgressHandler; @@ -89,7 +91,7 @@ public class ResumeUploaderFast implements Runnable { /** * 上传域名 */ - private String upHost; + volatile String upHost; /** * 总块数 */ @@ -97,11 +99,11 @@ public class ResumeUploaderFast implements Runnable { /** * 重传域名数 */ - private int retried = 0; + volatile int retried = 0; /** * 单域名检测次数 */ - private int singleDomainRetry = 1; + volatile int singleDomainRetry = 1; /** * 线程数量 */ @@ -120,6 +122,10 @@ public class ResumeUploaderFast implements Runnable { */ private int upBlock = 0; private final int domainRetry = 3; + /** + * 避免多个任务同时回调 + */ + private boolean isInterrupted = false; ResumeUploaderFast(Client client, Configuration config, File f, String key, UpToken token, final UpCompletionHandler completionHandler, UploadOptions options, String recorderKey, int multithread) { @@ -142,12 +148,19 @@ public void complete(String key, ResponseInfo info, JSONObject response) { e.printStackTrace(); } } - completionHandler.complete(key, info, response); + //第一个回调出来之后通知其他线程停止这个事件的回调 + synchronized (this) { + if (!isInterrupted) { + isInterrupted = true; + completionHandler.complete(key, info, response); + } else { + return; + } + } } }; this.options = options != null ? options : UploadOptions.defaultOptions(); tblock = (int) (totalSize + Configuration.BLOCK_SIZE - 1) / Configuration.BLOCK_SIZE; - ; this.offsets = new Long[tblock]; contexts = new String[tblock]; modifyTime = f.lastModified(); @@ -165,9 +178,15 @@ public void run() { return; } putBlockInfo(); + upHost = config.zone.upHost(token.token, config.useHttps, null); - BlockElement fblock = getBlockInfo(); - mkblk(fblock.getOffset(), fblock.getBlocksize(), upHost); + if (blockInfo.size() < multithread) { + multithread = blockInfo.size(); + } + for (int i = 0; i < multithread; i++) { + BlockElement mblock = getBlockInfo(); + new UploadTread(mblock.getOffset(), mblock.getBlocksize(), upHost).start(); + } } @@ -176,7 +195,6 @@ public void run() { */ private void putBlockInfo() { Long[] offs = recoveryFromRecord(); - int lastBlock = tblock - 1; if (offs == null) { for (int i = 0; i < lastBlock; i++) { @@ -250,12 +268,15 @@ public void run() { private void mkblk(long offset, int blockSize, String upHost) { String path = format(Locale.ENGLISH, "/mkblk/%d", blockSize); byte[] chunkBuffer = new byte[blockSize]; - try { - file.seek(offset); - file.read(chunkBuffer, 0, blockSize); - } catch (IOException e) { - completionHandler.complete(key, ResponseInfo.fileError(e, token), null); - return; + synchronized (this) { + try { + //多线程时file.read可能会在seek时被篡改 + file.seek(offset); + file.read(chunkBuffer, 0, blockSize); + } catch (IOException e) { + completionHandler.complete(key, ResponseInfo.fileError(e, token), null); + return; + } } long crc32 = Crc32.bytes(chunkBuffer, 0, blockSize); @@ -345,6 +366,7 @@ public void complete(ResponseInfo info, JSONObject response) { }; } + private CompletionHandler getCompletionHandler(final long offset, final int blockSize, final long crc32) { return new CompletionHandler() { @Override @@ -366,19 +388,22 @@ public void complete(ResponseInfo info, JSONObject response) { //上传失败：重试 //701: ctx不正确或者已经过期 + //checkRetried之后应该立即updateRetried，否则每个线程进来checkRetried判定都是跟第一个一样 if (!isChunkOK(info, response)) { if (info.statusCode == 701 && checkRetried()) { - mkblk(offset, blockSize, upHost); + updateRetried(); + mkblk(offset, blockSize, upHost); return; } if (upHost != null && ((isNotChunkToQiniu(info, response) || info.needRetry()) && checkRetried())) { - mkblk(offset, blockSize, upHost); updateRetried(); + mkblk(offset, blockSize, upHost); return; } + completionHandler.complete(key, info, response); return; } @@ -387,8 +412,9 @@ && checkRetried())) { String context = null; if (response == null && checkRetried()) { - mkblk(offset, blockSize, upHost); updateRetried(); + mkblk(offset, blockSize, upHost); + return; } long crc = 0; @@ -401,9 +427,8 @@ && checkRetried())) { e.printStackTrace(); } if ((context == null || crc != crc32) && checkRetried()) { - retried += 1; - mkblk(offset, blockSize, upHost); updateRetried(); + mkblk(offset, blockSize, upHost); return; } if (context == null) { @@ -434,17 +459,7 @@ && checkRetried())) { return; } - if (isFirstTask) { - if (blockInfo.size() < multithread) { - multithread = blockInfo.size(); - } - for (int i = 0; i < multithread; i++) { - BlockElement mblock = getBlockInfo(); - Thread t = new UploadTread(mblock.getOffset(), mblock.getBlocksize(), upHost); - t.start(); - } - isFirstTask = false; - } else { + if (blockInfo.size() > 0) { BlockElement mblock = getBlockInfo(); if (mblock.getOffset() != 0 && mblock.getBlocksize() != 0) new UploadTread(mblock.getOffset(), mblock.getBlocksize(), upHost).start(); @@ -461,9 +476,10 @@ private synchronized void updateRetried() { retried += 1; upHost = config.zone.upHost(token.token, config.useHttps, upHost); } + } - private boolean checkRetried() { + private synchronized boolean checkRetried() { return retried < domainRetry; } diff --git a/library/src/main/java/com/qiniu/android/utils/StringUtils.java b/library/src/main/java/com/qiniu/android/utils/StringUtils.java index 9591bea3f..53ae5ac2b 100644 --- a/library/src/main/java/com/qiniu/android/utils/StringUtils.java +++ b/library/src/main/java/com/qiniu/android/utils/StringUtils.java @@ -1,5 +1,7 @@ package com.qiniu.android.utils; +import android.util.Log; + import com.qiniu.android.common.Constants; import java.io.UnsupportedEncodingException; @@ -69,6 +71,9 @@ public static String join(String[] array, String sep) { */ public static String jsonJoin(String[] array) { int arraySize = array.length; + if (array[0] == null) { + array[0] = ""; + } int bufSize = arraySize * (array[0].length() + 3); StringBuilder buf = new StringBuilder(bufSize); for (int i = 0; i < arraySize; i++) { @@ -89,15 +94,12 @@ public static String jsonJoin(Long[] array) { } public static String[] longToString(Long longArray[]) { - if (longArray == null || longArray.length < 1) { - return null; - } String stringArray[] = new String[longArray.length]; for (int i = 0; i < stringArray.length; i++) { try { stringArray[i] = String.valueOf(longArray[i]); } catch (NumberFormatException e) { - stringArray[i] = null; + stringArray[i] = "null"; continue; } } From d3d7561f281281e746e606d8d25ac2eae2006da7 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Thu, 1 Aug 2019 11:36:11 +0800 Subject: [PATCH 08/31] format code --- .../android/storage/ResumeUploaderFast.java | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java index 8ce843b09..41ec99634 100644 --- a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java +++ b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java @@ -1,7 +1,5 @@ package com.qiniu.android.storage; -import android.util.Log; - import com.qiniu.android.http.Client; import com.qiniu.android.http.CompletionHandler; import com.qiniu.android.http.ProgressHandler; @@ -33,13 +31,6 @@ * Created by jemy on 2019/7/9. */ -/** - * 线程不安全：contexts，crc32，getBlockInfo() - * 需要它不安全的有：upHost - * progress回调不能根据offset，要根据已经上传的总大小 - * 修正了preQurey err - * update AutoZone construct to query zone use http or https - */ public class ResumeUploaderFast implements Runnable { /** @@ -201,7 +192,7 @@ private void putBlockInfo() { blockInfo.put((long) i * Configuration.BLOCK_SIZE, Configuration.BLOCK_SIZE); } blockInfo.put((long) lastBlock * Configuration.BLOCK_SIZE, (int) (totalSize - lastBlock * Configuration.BLOCK_SIZE)); - } else {//有断点文件 + } else { HashSet set = new HashSet(Arrays.asList(offs)); for (int i = 0; i < lastBlock; i++) { Long offset = (long) i * Configuration.BLOCK_SIZE; @@ -225,7 +216,7 @@ private void putBlockInfo() { /** * get next block use to upload * - * @return + * @return BlockElement */ private synchronized BlockElement getBlockInfo() { Iterator> it = blockInfo.entrySet().iterator(); @@ -270,7 +261,7 @@ private void mkblk(long offset, int blockSize, String upHost) { byte[] chunkBuffer = new byte[blockSize]; synchronized (this) { try { - //多线程时file.read可能会在seek时被篡改 + //多线程时file.read可能会在seek时被篡改，导致上传buffer紊乱 file.seek(offset); file.read(chunkBuffer, 0, blockSize); } catch (IOException e) { @@ -391,7 +382,6 @@ public void complete(ResponseInfo info, JSONObject response) { //checkRetried之后应该立即updateRetried，否则每个线程进来checkRetried判定都是跟第一个一样 if (!isChunkOK(info, response)) { if (info.statusCode == 701 && checkRetried()) { - updateRetried(); mkblk(offset, blockSize, upHost); return; @@ -410,7 +400,6 @@ && checkRetried())) { //上传成功(伪成功)：检查response String context = null; - if (response == null && checkRetried()) { updateRetried(); mkblk(offset, blockSize, upHost); @@ -571,5 +560,4 @@ public int getBlocksize() { } } - } From 3455c491ffb1b0df0d9d36d54a85f5caabe1f8e1 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Thu, 1 Aug 2019 17:00:21 +0800 Subject: [PATCH 09/31] update multithreads --- .../src/main/java/com/qiniu/android/storage/UploadManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/src/main/java/com/qiniu/android/storage/UploadManager.java b/library/src/main/java/com/qiniu/android/storage/UploadManager.java index fe7e03770..119e87777 100644 --- a/library/src/main/java/com/qiniu/android/storage/UploadManager.java +++ b/library/src/main/java/com/qiniu/android/storage/UploadManager.java @@ -21,7 +21,7 @@ public final class UploadManager { private final Configuration config; private final Client client; - private int multithreads = 0; + private int multithreads = 1; private static int DEF_THREAD_NUM = 3; /** From b6e3beb34dc3cab89a5b7133926e6ec393cf49e9 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Fri, 2 Aug 2019 10:59:18 +0800 Subject: [PATCH 10/31] protect mkfile func --- .../main/java/com/qiniu/android/storage/ResumeUploaderFast.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java index 41ec99634..f600f1408 100644 --- a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java +++ b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java @@ -443,7 +443,7 @@ && checkRetried())) { record(offsets); upBlock += 1; } - if (upBlock >= tblock) { + if (upBlock == tblock) { makeFile(upHost, getMkfileCompletionHandler(), options.cancellationSignal); return; } From 053f72ef7d3884523640d12e9de7799594a366c1 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Fri, 2 Aug 2019 11:41:38 +0800 Subject: [PATCH 11/31] Thread --- .../com/qiniu/android/storage/ResumeUploaderFast.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java index f600f1408..216fc2901 100644 --- a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java +++ b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java @@ -176,7 +176,7 @@ public void run() { } for (int i = 0; i < multithread; i++) { BlockElement mblock = getBlockInfo(); - new UploadTread(mblock.getOffset(), mblock.getBlocksize(), upHost).start(); + new UploadThread(mblock.getOffset(), mblock.getBlocksize(), upHost).start(); } } @@ -231,12 +231,12 @@ private synchronized BlockElement getBlockInfo() { return new BlockElement(offset, blockSize); } - class UploadTread extends Thread { + class UploadThread extends Thread { private long offset; private int blockSize; private String upHost; - UploadTread(long offset, int blockSize, String upHost) { + UploadThread(long offset, int blockSize, String upHost) { this.offset = offset; this.blockSize = blockSize; this.upHost = upHost; @@ -451,7 +451,7 @@ && checkRetried())) { if (blockInfo.size() > 0) { BlockElement mblock = getBlockInfo(); if (mblock.getOffset() != 0 && mblock.getBlocksize() != 0) - new UploadTread(mblock.getOffset(), mblock.getBlocksize(), upHost).start(); + new UploadThread(mblock.getOffset(), mblock.getBlocksize(), upHost).start(); } } }; From ee7da7a7ab58579141cd6c673a1a9682db450652 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Mon, 9 Sep 2019 20:47:42 +0800 Subject: [PATCH 12/31] dnsPrefetch --- library/library.iml | 1 + .../java/com/qiniu/android/DnsApiTest.java | 175 +++++++++++ .../com/qiniu/android/ResumeUploadTest.java | 2 +- .../java/com/qiniu/android/TestCompany.java | 30 ++ .../java/com/qiniu/android/TestConfig.java | 5 +- .../qiniu/android/common/AutoZoneTest.java | 7 +- .../com/qiniu/android/collect/Config.java | 10 + .../com/qiniu/android/common/AutoZone.java | 7 + .../com/qiniu/android/common/FixedZone.java | 37 ++- .../com/qiniu/android/common/ZoneInfo.java | 2 + .../java/com/qiniu/android/http/Client.java | 28 +- .../com/qiniu/android/http/DnsPrefetcher.java | 279 ++++++++++++++++++ .../com/qiniu/android/http/ResponseInfo.java | 2 +- .../qiniu/android/storage/Configuration.java | 13 + .../qiniu/android/storage/UploadManager.java | 136 ++++++++- .../qiniu/android/utils/AndroidNetwork.java | 38 +++ .../com/qiniu/android/utils/StringUtils.java | 69 +++++ 17 files changed, 817 insertions(+), 24 deletions(-) create mode 100644 library/src/androidTest/java/com/qiniu/android/DnsApiTest.java create mode 100644 library/src/androidTest/java/com/qiniu/android/TestCompany.java create mode 100644 library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java diff --git a/library/library.iml b/library/library.iml index d715e051f..d44595374 100644 --- a/library/library.iml +++ b/library/library.iml @@ -99,6 +99,7 @@ + diff --git a/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java b/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java new file mode 100644 index 000000000..3b9e5d3d0 --- /dev/null +++ b/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java @@ -0,0 +1,175 @@ +package com.qiniu.android; + +import android.graphics.LinearGradient; +import android.test.AndroidTestCase; +import android.test.InstrumentationTestCase; +import android.util.Log; + +import com.qiniu.android.collect.Config; +import com.qiniu.android.common.ZoneInfo; +import com.qiniu.android.http.DnsPrefetcher; +import com.qiniu.android.storage.Recorder; +import com.qiniu.android.storage.UploadManager; +import com.qiniu.android.storage.persistent.FileRecorder; +import com.qiniu.android.utils.AndroidNetwork; +import com.qiniu.android.utils.StringUtils; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.IOException; +import java.io.Serializable; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; + +/** + * Created by jemy on 2019/8/20. + */ + +public class DnsApiTest extends InstrumentationTestCase { + public void testDns() throws Throwable { + List inetAddresses = null; + DnsPrefetcher dnsPrefetcher; +// try { +// inetAddresses = DnsPrefetcher.getDnsBySystem().lookup("upload.qiniup.com"); +// } catch (UnknownHostException e) { +// e.printStackTrace(); +// } + Log.e("qiniutest", "InetAddress: " + inetAddresses.size()); + //超耗时过程 +// for (int i = 0; i < inetAddresses.size(); i++) { +// Log.e("qiniutest", "InetAddress.getCanonicalHostName: " + inetAddresses.get(i).getCanonicalHostName()); +// +// } + for (int i = 0; i < inetAddresses.size(); i++) { + Log.e("qiniutest", "InetAddress.getHostAddress: " + inetAddresses.get(i).getHostAddress()); + } + } + + + public void testQueryDomain() { + ZoneInfo info = null; + + DnsPrefetcher dnsPrefetcher = DnsPrefetcher.getDnsPrefetcher(); + try { + info = dnsPrefetcher.init(TestConfig.uptoken_prefetch).getPreQueryZone(); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + if (info == null) { + Log.e("qiniutest: ", "null"); + } + Log.e("qiniutest: ", info.toString()); + Log.e("qiniutest: ", info.upDomainsList.get(0)); + } + + + public void testLocalDomain() { + List info = null; + DnsPrefetcher dnsPrefetcher = DnsPrefetcher.getDnsPrefetcher(); + try { + info = dnsPrefetcher.init(TestConfig.uptoken_prefetch).getLocalZone(); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + if (info == null) { + Log.e("qiniutest: ", "null"); + } + for (int i = 0; i < info.size(); i++) { + Log.e("qiniutest: ", info.get(i).toString()); + } + + } + + + public void testLocalIp() { + String s = AndroidNetwork.getHostIP(); + Log.e("qiniutest", s); + } + + + public void testSerializeCache() { + String recordKey = "/sdcard/dnschache"; + try { + Recorder recorder = new FileRecorder(Config.dnscacheDir); + + String s = String.valueOf(System.currentTimeMillis()); + TestCompany company = new TestCompany("qiniu", 8); + byte[] com = StringUtils.toByteArray(company); + Log.e("qiniutest", s); + recorder.set("time", s.getBytes()); + recorder.set("compant", com); + + try { + Thread.sleep(900); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + byte[] time = recorder.get("time"); + byte[] getcom = recorder.get("compant"); + Log.e("qiniutest", new String(time)); + TestCompany company1 = (TestCompany) StringUtils.toObject(getcom); + Log.e("qiniutest", "name: " + company1.getName() + " ,age: " + company1.getAge()); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void testDnsPreAndcache() { + UploadManager up = new UploadManager(); + boolean needPrefetch = up.checkRePrefetchDns(Config.dnscacheDir); + Log.e("qiniutest", "check:" + needPrefetch); + if (needPrefetch) { + up.startPrefetchDns(TestConfig.uptoken_prefetch); + } else { + testRecoverCache(); + return; + } + //预取或者recover success + List list = DnsPrefetcher.getDnsPrefetcher().getHosts(); + HashMap> map = DnsPrefetcher.getDnsPrefetcher().getConcurrentHashMap(); + Log.e("qiniutest: ", "list size: " + list.size()); + for (String s : list) { + Log.e("qiniutest: ", "uphost: " + s); + List list1 = map.get(s); + for (InetAddress inetAddress : + list1) { + Log.e("qiniutest: ", "ip: " + inetAddress.getHostAddress()); + } + } + + } + + //test recover + public void testRecoverCache() { + + Recorder recorder = null; + try { + recorder = new FileRecorder(Config.dnscacheDir); + } catch (IOException e) { + e.printStackTrace(); + } + new UploadManager().recoverDnsCache(recorder); + + + HashMap> map1 = DnsPrefetcher.getDnsPrefetcher().getConcurrentHashMap(); + List list = DnsPrefetcher.getDnsPrefetcher().getHosts(); + for (String s : list) { + Log.e("qiniutest: ", "uphost for cache: " + s); + List list1 = map1.get(s); + for (InetAddress inetAddress : + list1) { + Log.e("qiniutest: ", "ip for cache: " + inetAddress.getHostAddress()); + } + } + } + + +} diff --git a/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java b/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java index 57b71f859..e70791bce 100644 --- a/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java +++ b/library/src/androidTest/java/com/qiniu/android/ResumeUploadTest.java @@ -192,7 +192,7 @@ public void test4M1k2() throws Throwable { @LargeTest public void testResumeUploadFast() throws Throwable { - template(1024 * 98); + template(1024 * 8); } // @LargeTest diff --git a/library/src/androidTest/java/com/qiniu/android/TestCompany.java b/library/src/androidTest/java/com/qiniu/android/TestCompany.java new file mode 100644 index 000000000..59c6dcb0c --- /dev/null +++ b/library/src/androidTest/java/com/qiniu/android/TestCompany.java @@ -0,0 +1,30 @@ +package com.qiniu.android; + +import java.io.Serializable; + +/** + * Created by jemy on 2019/9/5. + */ + +public class TestCompany implements Serializable { + String name; + int age; + + public TestCompany(String name, int age) { + this.name = name; + this.age = age; + } + + public String getName() { + return name; + } + + public int getAge() { + return age; + } + + @Override + public String toString() { + return super.toString(); + } +} diff --git a/library/src/androidTest/java/com/qiniu/android/TestConfig.java b/library/src/androidTest/java/com/qiniu/android/TestConfig.java index 2deaa2dac..b9509410c 100644 --- a/library/src/androidTest/java/com/qiniu/android/TestConfig.java +++ b/library/src/androidTest/java/com/qiniu/android/TestConfig.java @@ -9,7 +9,7 @@ public final class TestConfig { //华东上传凭证 public static final String bucket_z0 = "sdk-z0"; - public static final String token_z0 = "QWYn5TFQsLLU1pL5MFEmX3s5DmHdUThav9WyOWOm:KUw9Fxq8NoLTeZNsKkhTjoKpsuQ=:eyJzY29wZSI6InNkay16MCIsInJldHVybmJvZHkiOiJ7XCJoYXNoXCI6XCIkKGV0YWcpXCIsXCJrZXlcIjpcIiQoa2V5KVwiLFwiZm5hbWVcIjpcIiAkKGZuYW1lKSBcIixcImZzaXplXCI6XCIkKGZzaXplKVwiLFwibWltZVR5cGVcIjpcIiQobWltZVR5cGUpXCIsXCJmb29cIjpcIiQoeDpmb28pXCIsXCJiYXJcIjpcIiQoeDpiYXIpXCJ9IiwiZGVhZGxpbmUiOjQyOTQ5NjcyOTV9"; + public static final String token_z0 = "MP_Ebql_lSsUrDr7WrXn_5vKocQDLvTPCNEFeVmp:zfCfgMtCzO8l8iab_lbm402wZY8=:eyJzY29wZSI6ImFuZHJvaWR0ZXN0IiwiZGVhZGxpbmUiOjE1Njk4NTgzMTl9"; //华北上传凭证 public static final String bucket_z1 = "sdk-z1"; public static final String token_z1 = "QWYn5TFQsLLU1pL5MFEmX3s5DmHdUThav9WyOWOm:ILIWwUZ7_hvZeKJzbBKvpo8DpYc=:eyJzY29wZSI6InNkay16MSIsInJldHVybmJvZHkiOiJ7XCJoYXNoXCI6XCIkKGV0YWcpXCIsXCJrZXlcIjpcIiQoa2V5KVwiLFwiZm5hbWVcIjpcIiAkKGZuYW1lKSBcIixcImZzaXplXCI6XCIkKGZzaXplKVwiLFwibWltZVR5cGVcIjpcIiQobWltZVR5cGUpXCIsXCJmb29cIjpcIiQoeDpmb28pXCIsXCJiYXJcIjpcIiQoeDpiYXIpXCJ9IiwiZGVhZGxpbmUiOjQyOTQ5NjcyOTV9"; @@ -21,6 +21,9 @@ public final class TestConfig { public static final String token_na0 = "QWYn5TFQsLLU1pL5MFEmX3s5DmHdUThav9WyOWOm:QxnYkUaYIw4fIsI133_tjFJ_03M=:eyJzY29wZSI6InNkay1uYTAiLCJyZXR1cm5ib2R5Ijoie1wiaGFzaFwiOlwiJChldGFnKVwiLFwia2V5XCI6XCIkKGtleSlcIixcImZuYW1lXCI6XCIgJChmbmFtZSkgXCIsXCJmc2l6ZVwiOlwiJChmc2l6ZSlcIixcIm1pbWVUeXBlXCI6XCIkKG1pbWVUeXBlKVwiLFwiZm9vXCI6XCIkKHg6Zm9vKVwiLFwiYmFyXCI6XCIkKHg6YmFyKVwifSIsImRlYWRsaW5lIjo0Mjk0OTY3Mjk1fQ=="; public static final String ak = "QWYn5TFQsLLU1pL5MFEmX3s5DmHdUThav9WyOWOm"; + //dns prefetch token + public static final String uptoken_prefetch = "MP_Ebql_lSsUrDr7WrXn_5vKocQDLvTPCNEFeVmp:3KJpXCGMqm6EAYU71RF1HDmQrcE=:eyJzY29wZSI6ImFuZHJvaWR0ZXN0IiwiZGVhZGxpbmUiOjE1Njc0OTAxODF9"; + /** * 华东机房 */ diff --git a/library/src/androidTest/java/com/qiniu/android/common/AutoZoneTest.java b/library/src/androidTest/java/com/qiniu/android/common/AutoZoneTest.java index 065400496..58ff5659a 100644 --- a/library/src/androidTest/java/com/qiniu/android/common/AutoZoneTest.java +++ b/library/src/androidTest/java/com/qiniu/android/common/AutoZoneTest.java @@ -1,11 +1,15 @@ package com.qiniu.android.common; import android.test.AndroidTestCase; +import android.util.Log; import com.qiniu.android.TestConfig; +import com.qiniu.android.http.DnsPrefetcher; import junit.framework.Assert; +import java.net.UnknownHostException; +import java.util.List; import java.util.concurrent.CountDownLatch; /** @@ -76,10 +80,11 @@ public void onFailure(int reason) { e.printStackTrace(); } ZoneInfo info = autoZone.zoneInfo(ak, bkt); -// Log.d("zone0: ", info.toString()); + Log.d("qiniutest: ", info.toString()); ZoneInfo info2 = autoZone.zoneInfo(ak, bkt); Assert.assertSame(info, info2); } + } diff --git a/library/src/main/java/com/qiniu/android/collect/Config.java b/library/src/main/java/com/qiniu/android/collect/Config.java index 0e2707f76..7793a193d 100644 --- a/library/src/main/java/com/qiniu/android/collect/Config.java +++ b/library/src/main/java/com/qiniu/android/collect/Config.java @@ -71,6 +71,16 @@ public final class Config { */ public static int interval = 10; + /** + * dns缓存信息目录 + */ + public static String dnscacheDir = "/sdcard/dnschache"; + + /** + * + */ + public static String preQueryHost = "uc.qbox.me"; + /** * 当网络切换到 wifi 下，切换到此设置 */ diff --git a/library/src/main/java/com/qiniu/android/common/AutoZone.java b/library/src/main/java/com/qiniu/android/common/AutoZone.java index 102bb72d0..71cc82a04 100644 --- a/library/src/main/java/com/qiniu/android/common/AutoZone.java +++ b/library/src/main/java/com/qiniu/android/common/AutoZone.java @@ -4,6 +4,7 @@ import com.qiniu.android.http.CompletionHandler; import com.qiniu.android.http.ResponseInfo; import com.qiniu.android.storage.UpToken; +import com.qiniu.android.utils.StringMap; import com.qiniu.android.utils.UrlSafeBase64; import org.json.JSONException; @@ -24,6 +25,7 @@ public final class AutoZone extends Zone { private final String ucServer; private Map zones = new ConcurrentHashMap<>(); private Client client = new Client(); + private ZoneInfo zoneInfo; /** * default useHttps to req autoZone @@ -81,6 +83,7 @@ void preQueryIndex(final ZoneIndex index, final QueryHandler complete) { return; } ZoneInfo info = zones.get(index); + zoneInfo = info; if (info != null) { complete.onSuccess(); return; @@ -92,6 +95,7 @@ public void complete(ResponseInfo info, JSONObject response) { if (info.isOK() && response != null) { try { ZoneInfo info2 = ZoneInfo.buildFromJson(response); + zoneInfo = info2; zones.put(index, info2); complete.onSuccess(); return; @@ -112,11 +116,13 @@ boolean preQueryIndex(final ZoneIndex index) { if (index != null) { ZoneInfo info = zones.get(index); if (info != null) { + zoneInfo = info; success = true; } else { try { ResponseInfo responseInfo = getZoneJsonSync(index); ZoneInfo info2 = ZoneInfo.buildFromJson(responseInfo.response); + zoneInfo = info2; zones.put(index, info2); success = true; } catch (JSONException e) { @@ -127,6 +133,7 @@ boolean preQueryIndex(final ZoneIndex index) { return success; } + @Override public synchronized String upHost(String token, boolean useHttps, String frozenDomain) { ZoneInfo info = queryByToken(token); diff --git a/library/src/main/java/com/qiniu/android/common/FixedZone.java b/library/src/main/java/com/qiniu/android/common/FixedZone.java index 9d5e62162..6c89201c1 100644 --- a/library/src/main/java/com/qiniu/android/common/FixedZone.java +++ b/library/src/main/java/com/qiniu/android/common/FixedZone.java @@ -4,6 +4,7 @@ import java.net.URI; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -16,49 +17,64 @@ public final class FixedZone extends Zone { /** * 华东机房 */ - public static final Zone zone0 = new FixedZone(new String[]{ + static String[] arrayzone0 = new String[]{ "upload.qiniup.com", "upload-jjh.qiniup.com", "upload-xs.qiniup.com", "up.qiniup.com", "up-jjh.qiniup.com", "up-xs.qiniup.com", "upload.qbox.me", "up.qbox.me" - }); + }; + public static final Zone zone0 = new FixedZone(arrayzone0); /** * 华北机房 */ - public static final Zone zone1 = new FixedZone(new String[]{ + static String[] arrayzone1 = new String[]{ "upload-z1.qiniup.com", "up-z1.qiniup.com", "upload-z1.qbox.me", "up-z1.qbox.me" - }); + }; + public static final Zone zone1 = new FixedZone(arrayzone1); /** * 华南机房 */ - public static final Zone zone2 = new FixedZone(new String[]{ + static String[] arrayzone2 = new String[]{ "upload-z2.qiniup.com", "upload-dg.qiniup.com", "upload-fs.qiniup.com", "up-z2.qiniup.com", "up-dg.qiniup.com", "up-fs.qiniup.com", "upload-z2.qbox.me", "up-z2.qbox.me" - }); + }; + public static final Zone zone2 = new FixedZone(arrayzone2); /** * 北美机房 */ - public static final Zone zoneNa0 = new FixedZone(new String[]{ + static String[] arrayzoneNa0 = new String[]{ "upload-na0.qiniup.com", "up-na0.qiniup.com", "upload-na0.qbox.me", "up-na0.qbox.me" - }); + }; + public static final Zone zoneNa0 = new FixedZone(arrayzoneNa0); /** * 新加坡机房 */ - public static final Zone zoneAs0 = new FixedZone(new String[]{ + static String[] arrayZoneAs0 = new String[]{ "upload-as0.qiniup.com", "up-as0.qiniup.com", "upload-as0.qbox.me", "up-as0.qbox.me" - }); + }; + public static final Zone zoneAs0 = new FixedZone(arrayZoneAs0); private ZoneInfo zoneInfo; + public static List getZoneInfos() { + List listZoneInfo = new ArrayList(); + listZoneInfo.add(createZoneInfo(arrayzone0)); + listZoneInfo.add(createZoneInfo(arrayzone1)); + listZoneInfo.add(createZoneInfo(arrayzone2)); + listZoneInfo.add(createZoneInfo(arrayzoneNa0)); + listZoneInfo.add(createZoneInfo(arrayZoneAs0)); + return listZoneInfo; + } + public FixedZone(ZoneInfo zoneInfo) { this.zoneInfo = zoneInfo; } @@ -77,6 +93,7 @@ public static ZoneInfo createZoneInfo(String[] upDomains) { return new ZoneInfo(0, upDomainsList, upDomainsMap); } + @Override public synchronized String upHost(String upToken, boolean useHttps, String frozenDomain) { String upHost = this.upHost(this.zoneInfo, useHttps, frozenDomain); diff --git a/library/src/main/java/com/qiniu/android/common/ZoneInfo.java b/library/src/main/java/com/qiniu/android/common/ZoneInfo.java index 2e2fd72c3..d9b6959aa 100644 --- a/library/src/main/java/com/qiniu/android/common/ZoneInfo.java +++ b/library/src/main/java/com/qiniu/android/common/ZoneInfo.java @@ -1,5 +1,7 @@ package com.qiniu.android.common; +import android.util.Log; + import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; diff --git a/library/src/main/java/com/qiniu/android/http/Client.java b/library/src/main/java/com/qiniu/android/http/Client.java index a2d4122a6..d5454b10b 100644 --- a/library/src/main/java/com/qiniu/android/http/Client.java +++ b/library/src/main/java/com/qiniu/android/http/Client.java @@ -1,8 +1,13 @@ package com.qiniu.android.http; +import android.util.Log; + +import com.qiniu.android.collect.Config; import com.qiniu.android.common.Constants; +import com.qiniu.android.storage.Recorder; import com.qiniu.android.storage.UpCancellationSignal; import com.qiniu.android.storage.UpToken; +import com.qiniu.android.storage.persistent.FileRecorder; import com.qiniu.android.utils.AsyncRun; import com.qiniu.android.utils.StringMap; import com.qiniu.android.utils.StringUtils; @@ -13,6 +18,8 @@ import java.net.InetAddress; import java.net.SocketTimeoutException; import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.concurrent.TimeUnit; @@ -47,26 +54,29 @@ public Client() { public Client(ProxyConfiguration proxy, int connectTimeout, int responseTimeout, UrlConverter converter, final Dns dns) { this.converter = converter; OkHttpClient.Builder builder = new OkHttpClient.Builder(); - if (proxy != null) { builder.proxy(proxy.proxy()); if (proxy.user != null && proxy.password != null) { builder.proxyAuthenticator(proxy.authenticator()); } } - if (dns != null) { - builder.dns(new okhttp3.Dns() { - @Override - public List lookup(String hostname) throws UnknownHostException { + + builder.dns(new okhttp3.Dns() { + @Override + public List lookup(String hostname) throws UnknownHostException { + if (dns != null) { try { return dns.lookup(hostname); } catch (Exception e) { e.printStackTrace(); } - return okhttp3.Dns.SYSTEM.lookup(hostname); + } else if (DnsPrefetcher.getDnsPrefetcher().getInetAddressByHost(hostname) != null) { + return DnsPrefetcher.getDnsPrefetcher().getInetAddressByHost(hostname); } - }); - } + return okhttp3.Dns.SYSTEM.lookup(hostname); + } + }); + builder.networkInterceptors().add(new Interceptor() { @Override public okhttp3.Response intercept(Chain chain) throws IOException { @@ -434,7 +444,7 @@ public void accept(String key, Object value) { } } - private static class ResponseTag { + public static class ResponseTag { public String ip = ""; public long duration = -1; } diff --git a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java new file mode 100644 index 000000000..3f6e37134 --- /dev/null +++ b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java @@ -0,0 +1,279 @@ +package com.qiniu.android.http; + +import com.qiniu.android.collect.Config; +import com.qiniu.android.common.Constants; +import com.qiniu.android.common.FixedZone; +import com.qiniu.android.common.Zone; +import com.qiniu.android.common.ZoneInfo; +import com.qiniu.android.storage.UpToken; +import com.qiniu.android.utils.UrlSafeBase64; + +import org.json.JSONException; +import org.json.JSONObject; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +/** + * + *

+ * Created by jemy on 2019/8/20. + */ + +public class DnsPrefetcher { + + public static DnsPrefetcher dnsPrefetcher = null; + private String token; + + private HashMap> mConcurrentHashMap = new HashMap>(); + private List mHosts = new ArrayList(); + + + private DnsPrefetcher() { + + } + + public void setConcurrentHashMap(HashMap> mConcurrentHashMap) { + this.mConcurrentHashMap = mConcurrentHashMap; + } + + public DnsPrefetcher init(String token) throws UnknownHostException { + this.token = token; + preHosts(); + preFetch(); + return this; + } + + + //use for test + public List getHosts() { + return this.mHosts; + } + + public void setHosts(List mHosts) { + this.mHosts = mHosts; + } + + //use for test + public HashMap> getConcurrentHashMap() { + return this.mConcurrentHashMap; + } + + //use for test + public void setToken(String token) { + this.token = token; + } + + public List getInetAddressByHost(String host) { + if (mConcurrentHashMap != null) { + return mConcurrentHashMap.get(host); + } + return null; + } + + public static DnsPrefetcher getDnsPrefetcher() { + if (dnsPrefetcher == null) { + dnsPrefetcher = new DnsPrefetcher(); + synchronized (dnsPrefetcher) { + if (dnsPrefetcher == null) { + dnsPrefetcher = new DnsPrefetcher(); + } + } + } + return dnsPrefetcher; + } + + public void preHosts() { + HashSet set = new HashSet(); + + //preQuery sync + ZoneInfo zoneInfo = getPreQueryZone(); + if (zoneInfo != null) { + for (int i = 0; i < zoneInfo.upDomainsList.size(); i++) { + if (set.add(zoneInfo.upDomainsList.get(i))) + mHosts.add(zoneInfo.upDomainsList.get(i)); + } + } + //local + List listZoneinfo = getLocalZone(); + for (ZoneInfo zone : listZoneinfo) { + for (int i = 0; i < zone.upDomainsList.size(); i++) { + if (set.add(zone.upDomainsList.get(i))) + mHosts.add(zone.upDomainsList.get(i)); + } + } + mHosts.add(Config.preQueryHost); + } + + + public void preFetch() throws UnknownHostException { + for (int i = 0; i < mHosts.size(); i++) { + List inetAddresses = getDnsBySystem().lookup(mHosts.get(i)); + mConcurrentHashMap.put(mHosts.get(i), inetAddresses); + } + } + + + /** + * 自定义dns预取 + * + * @param dns + * @return + * @throws UnknownHostException + */ + public void dnsPreByCustom(Dns dns) { + for (int i = 0; i < mHosts.size(); i++) { + List inetAddresses = null; + try { + inetAddresses = dns.lookup(mHosts.get(i)); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + mConcurrentHashMap.put(mHosts.get(i), inetAddresses); + } + } + + + /** + * 系统DNS解析预取 + * + * @return + * @throws UnknownHostException + */ + public Dns getDnsBySystem() throws UnknownHostException { + return new SystemDns(); + } + + + class SystemDns implements Dns { + @Override + public List lookup(String hostname) throws UnknownHostException { + return okhttp3.Dns.SYSTEM.lookup(hostname); + } + } + + + /** + * look local host + */ + public List getLocalZone() { + List listZoneInfo = FixedZone.getZoneInfos(); + return listZoneInfo; + } + + + /** + * query host sync + */ + public ZoneInfo getPreQueryZone() { + if (token == null) { + return null; + } + DnsPrefetcher.ZoneIndex index = DnsPrefetcher.ZoneIndex.getFromToken(token); + ZoneInfo zoneInfo = preQueryIndex(index); + return zoneInfo; + } + + ZoneInfo preQueryIndex(DnsPrefetcher.ZoneIndex index) { + ZoneInfo info = null; + try { + ResponseInfo responseInfo = getZoneJsonSync(index); + info = ZoneInfo.buildFromJson(responseInfo.response); + } catch (JSONException e) { + e.printStackTrace(); + } + return info; + } + + ResponseInfo getZoneJsonSync(DnsPrefetcher.ZoneIndex index) { + Client client = new Client(); + String address = "http://" + Config.preQueryHost + "/v2/query?ak=" + index.accessKey + "&bucket=" + index.bucket; + return client.syncGet(address, null); + } + + + /** + * query host async + */ + public void getPreQueryZone(String token, final Zone.QueryHandler complete) { + DnsPrefetcher.ZoneIndex index = DnsPrefetcher.ZoneIndex.getFromToken(token); + + preQueryIndex(index, new CompletionHandler() { + @Override + public void complete(ResponseInfo info, JSONObject response) { + if (info.isOK() && response != null) { + try { + ZoneInfo zoneInfo = ZoneInfo.buildFromJson(response); + for (int i = 0; i < zoneInfo.upDomainsList.size(); i++) { + mHosts.add(zoneInfo.upDomainsList.get(i)); + } + complete.onSuccess(); + return; + } catch (JSONException e) { + e.printStackTrace(); + complete.onFailure(ResponseInfo.NetworkError); + return; + } + } + } + }); + } + + ZoneInfo preQueryIndex(DnsPrefetcher.ZoneIndex index, CompletionHandler handler) { + ZoneInfo info = null; + try { + ResponseInfo responseInfo = getZoneJsonAsync(index, handler); + info = ZoneInfo.buildFromJson(responseInfo.response); + } catch (JSONException e) { + e.printStackTrace(); + } + return info; + } + + private ResponseInfo getZoneJsonAsync(ZoneIndex index, CompletionHandler handler) { + Client client = new Client(); + String address = "http://" + Config.preQueryHost + "/v2/query?ak=" + index.accessKey + "&bucket=" + index.bucket; + client.asyncGet(address, null, UpToken.NULL, handler); + return null; + } + + + static class ZoneIndex { + final String accessKey; + final String bucket; + + ZoneIndex(String accessKey, String bucket) { + this.accessKey = accessKey; + this.bucket = bucket; + } + + static DnsPrefetcher.ZoneIndex getFromToken(String token) { + String[] strings = token.split(":"); + String ak = strings[0]; + String policy = null; + try { + policy = new String(UrlSafeBase64.decode(strings[2]), Constants.UTF_8); + JSONObject obj = new JSONObject(policy); + String scope = obj.getString("scope"); + String bkt = scope.split(":")[0]; + return new DnsPrefetcher.ZoneIndex(ak, bkt); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public int hashCode() { + return accessKey.hashCode() * 37 + bucket.hashCode(); + } + + public boolean equals(Object obj) { + return obj == this || !(obj == null || !(obj instanceof DnsPrefetcher.ZoneIndex)) + && ((DnsPrefetcher.ZoneIndex) obj).accessKey.equals(accessKey) && ((DnsPrefetcher.ZoneIndex) obj).bucket.equals(bucket); + } + } +} diff --git a/library/src/main/java/com/qiniu/android/http/ResponseInfo.java b/library/src/main/java/com/qiniu/android/http/ResponseInfo.java index 47d738ff8..152f5ce37 100644 --- a/library/src/main/java/com/qiniu/android/http/ResponseInfo.java +++ b/library/src/main/java/com/qiniu/android/http/ResponseInfo.java @@ -200,7 +200,7 @@ public static ResponseInfo fileError(Exception e, final UpToken upToken) { } public static ResponseInfo networkError(int code, UpToken upToken) { - return create(null, code, "", "", "", "", "", "", 80, 0, 0, "Network error during preQuery", upToken, 0); + return create(null, code, "", "", "", "", "", "", 80, 0, 0, "Network error during preQuery, Please use http try again", upToken, 0); } public static boolean isStatusCodeForBrokenNetwork(int code) { diff --git a/library/src/main/java/com/qiniu/android/storage/Configuration.java b/library/src/main/java/com/qiniu/android/storage/Configuration.java index bcf9cd7f3..efbac2651 100644 --- a/library/src/main/java/com/qiniu/android/storage/Configuration.java +++ b/library/src/main/java/com/qiniu/android/storage/Configuration.java @@ -65,6 +65,11 @@ public final class Configuration { */ public boolean useHttps; + /** + * dns预取缓存时间 + */ + public long dnsCacheTimeMs; + private Configuration(Builder builder) { useHttps = builder.useHttps; @@ -82,6 +87,8 @@ private Configuration(Builder builder) { proxy = builder.proxy; + dnsCacheTimeMs = builder.dnsCacheTimeMs; + urlConverter = builder.urlConverter; AutoZone autoZone = null; if (builder.useHttps) { @@ -120,6 +127,7 @@ public static class Builder { private int retryMax = 3; private UrlConverter urlConverter = null; private Dns dns = null; + private long dnsCacheTimeMs = 60 * 60 * 24 * 1000; public Builder zone(Zone zone) { this.zone = zone; @@ -182,6 +190,11 @@ public Builder useHttps(boolean useHttps) { return this; } + public Builder dnsCacheTimeMs(long dnsCacheTimeMs) { + this.dnsCacheTimeMs = dnsCacheTimeMs; + return this; + } + public Configuration build() { return new Configuration(this); } diff --git a/library/src/main/java/com/qiniu/android/storage/UploadManager.java b/library/src/main/java/com/qiniu/android/storage/UploadManager.java index 119e87777..736942d9d 100644 --- a/library/src/main/java/com/qiniu/android/storage/UploadManager.java +++ b/library/src/main/java/com/qiniu/android/storage/UploadManager.java @@ -1,16 +1,35 @@ package com.qiniu.android.storage; +import android.content.SharedPreferences; +import android.util.Log; + import com.qiniu.android.collect.Config; import com.qiniu.android.collect.UploadInfoCollector; import com.qiniu.android.common.Zone; +import com.qiniu.android.common.ZoneInfo; import com.qiniu.android.http.Client; +import com.qiniu.android.http.DnsPrefetcher; import com.qiniu.android.http.ResponseInfo; +import com.qiniu.android.storage.persistent.FileRecorder; +import com.qiniu.android.utils.AndroidNetwork; import com.qiniu.android.utils.AsyncRun; import com.qiniu.android.utils.StringUtils; +import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; + +import static java.lang.String.format; /** * 七牛文件上传管理器 @@ -136,6 +155,15 @@ public void put(final byte[] data, final String key, final String token, return; } + if (checkRePrefetchDns(token)) { + new Thread(new Runnable() { + @Override + public void run() { + startPrefetchDns(token); + } + }).start(); + } + Zone z = config.zone; z.preQuery(token, new Zone.QueryHandler() { @Override @@ -177,13 +205,22 @@ public void put(String filePath, String key, String token, UpCompletionHandler c * @param complete 上传完成的后续处理动作 * @param options 上传数据的可选参数 */ - public void put(final File file, final String key, String token, final UpCompletionHandler complete, + public void put(final File file, final String key, final String token, final UpCompletionHandler complete, final UploadOptions options) { final UpToken decodedToken = UpToken.parse(token); if (areInvalidArg(key, null, file, token, decodedToken, complete)) { return; } + if (checkRePrefetchDns(token)) { + new Thread(new Runnable() { + @Override + public void run() { + startPrefetchDns(token); + } + }).start(); + } + Zone z = config.zone; z.preQuery(token, new Zone.QueryHandler() { @Override @@ -307,4 +344,101 @@ public void run() { } } + + /** + *

+ * ip changed, the network has changed + * ak:scope变化，prequery（v2）自动获取域名接口发生变化，存储区域可能变化 + * cacheTime>config.cacheTime（默认24H） + *

+ * + * @return true:重新预期并缓存, false:不需要重新预取和缓存 + */ + public boolean checkRePrefetchDns(String token) { + Recorder recorder = null; + try { + recorder = new FileRecorder(Config.dnscacheDir); + } catch (IOException e) { + e.printStackTrace(); + return false; + } + byte[] data = recorder.get("lastcache"); + if (data == null) { + return true; + } + String jsonStr = new String(data); + JSONObject obj; + try { + obj = new JSONObject(jsonStr); + } catch (JSONException e) { + e.printStackTrace(); + return true; + } + String time = obj.optString("time"); + String ip = obj.optString("ip"); + String ak = obj.optString("ak"); + + String currentTime = String.valueOf(System.currentTimeMillis()); + String localip = AndroidNetwork.getHostIP(); + String akAndScope = StringUtils.getAkAndScope(token); + + long cacheTime = (Long.parseLong(currentTime) - Long.parseLong(time)) / 1000; + if (!localip.equals(ip) || cacheTime > config.dnsCacheTimeMs || !akAndScope.equals(ak)) { + return true; + } + + return recoverDnsCache(recorder); + } + + /** + * start preFetchDns: Time-consuming operation, in a thread + * @param token + */ + public void startPrefetchDns(String token) { + String currentTime = String.valueOf(System.currentTimeMillis()); + String localip = AndroidNetwork.getHostIP(); + String akAndScope = StringUtils.getAkAndScope(token); + String data = format(Locale.ENGLISH, "{\"time\":%s,\"ip\":%s,\"ak\":%s}", currentTime, localip, akAndScope); + Recorder recorder = null; + DnsPrefetcher dnsPrefetcher = null; + try { + recorder = new FileRecorder(Config.dnscacheDir); + dnsPrefetcher = DnsPrefetcher.getDnsPrefetcher().init(token); + } catch (IOException e) { + e.printStackTrace(); + } + if (config.dns != null) { + DnsPrefetcher.getDnsPrefetcher().dnsPreByCustom(config.dns); + } + HashMap> concurrentHashMap = dnsPrefetcher.getConcurrentHashMap(); + byte[] dnscache = StringUtils.toByteArray(concurrentHashMap); + + recorder.set("lastcache", data.getBytes()); + recorder.set("dnscache", dnscache); + } + + /** + * + * @param recorder + * @return + */ + public boolean recoverDnsCache(Recorder recorder) { + HashMap> concurrentHashMap = (HashMap>) StringUtils.toObject(recorder.get("dnscache")); + if (concurrentHashMap == null) { + return true; + } + DnsPrefetcher.getDnsPrefetcher().setConcurrentHashMap(concurrentHashMap); + + ArrayList list = new ArrayList(); + Iterator iter = concurrentHashMap.keySet().iterator(); + while (iter.hasNext()) { + String tmpkey = (String) iter.next(); + if (tmpkey == null || tmpkey.length() == 0) + continue; + list.add(tmpkey); + } + DnsPrefetcher.getDnsPrefetcher().setHosts(list); + return false; + } + } diff --git a/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java b/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java index c920070ae..e2ca717ca 100644 --- a/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java +++ b/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java @@ -3,6 +3,13 @@ import android.content.Context; import android.net.ConnectivityManager; import android.net.NetworkInfo; +import android.util.Log; + +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.util.Enumeration; /** * Created by bailong on 16/9/7. @@ -22,4 +29,35 @@ public static boolean isNetWorkReady() { return true; } } + + /** + * 获取ip地址 + * + * @return + */ + public static String getHostIP() { + String hostIp = null; + try { + Enumeration nis = NetworkInterface.getNetworkInterfaces(); + InetAddress ia = null; + while (nis.hasMoreElements()) { + NetworkInterface ni = (NetworkInterface) nis.nextElement(); + Enumeration ias = ni.getInetAddresses(); + while (ias.hasMoreElements()) { + ia = ias.nextElement(); + if (ia instanceof Inet6Address) { + continue;// skip ipv6 + } + String ip = ia.getHostAddress(); + if (!"127.0.0.1".equals(ip)) { + hostIp = ia.getHostAddress(); + break; + } + } + } + } catch (SocketException e) { + e.printStackTrace(); + } + return hostIp; + } } diff --git a/library/src/main/java/com/qiniu/android/utils/StringUtils.java b/library/src/main/java/com/qiniu/android/utils/StringUtils.java index 53ae5ac2b..1441e3e45 100644 --- a/library/src/main/java/com/qiniu/android/utils/StringUtils.java +++ b/library/src/main/java/com/qiniu/android/utils/StringUtils.java @@ -3,7 +3,15 @@ import android.util.Log; import com.qiniu.android.common.Constants; +import com.qiniu.android.http.DnsPrefetcher; +import org.json.JSONObject; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.io.UnsupportedEncodingException; /** @@ -133,4 +141,65 @@ public static String strip(String s) { } return b.toString(); } + + /** + * 对象转数组 + * + * @param obj + * @return + */ + public static byte[] toByteArray(Object obj) { + byte[] bytes = null; + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try { + ObjectOutputStream oos = new ObjectOutputStream(bos); + oos.writeObject(obj); + oos.flush(); + bytes = bos.toByteArray(); + oos.close(); + bos.close(); + } catch (IOException ex) { + ex.printStackTrace(); + } + return bytes; + } + + /** + * 数组转对象 + * + * @param bytes + * @return + */ + public static Object toObject(byte[] bytes) { + Object obj = null; + try { + ByteArrayInputStream bis = new ByteArrayInputStream(bytes); + ObjectInputStream ois = new ObjectInputStream(bis); + obj = ois.readObject(); + ois.close(); + bis.close(); + } catch (IOException ex) { + ex.printStackTrace(); + } catch (ClassNotFoundException ex) { + ex.printStackTrace(); + } + return obj; + } + + public static String getAkAndScope(String token) { + String[] strings = token.split(":"); + String ak = strings[0]; + String policy = null; + try { + policy = new String(UrlSafeBase64.decode(strings[2]), Constants.UTF_8); + JSONObject obj = new JSONObject(policy); + String scope = obj.getString("scope"); + String bkt = scope.split(":")[0]; + return ak + bkt; + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + } From ab61e438e363bdeaf0c83cc80c087dbe371bd503 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Mon, 9 Sep 2019 20:49:02 +0800 Subject: [PATCH 13/31] 7.3.18 --- library/src/main/java/com/qiniu/android/common/Constants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/src/main/java/com/qiniu/android/common/Constants.java b/library/src/main/java/com/qiniu/android/common/Constants.java index b66b2aec1..3a02de06f 100644 --- a/library/src/main/java/com/qiniu/android/common/Constants.java +++ b/library/src/main/java/com/qiniu/android/common/Constants.java @@ -2,7 +2,7 @@ public final class Constants { - public static final String VERSION = "7.3.17"; + public static final String VERSION = "7.3.18"; public static final String UTF_8 = "utf-8"; } From af002471cec413c6ecbb446d8b96cd398f1946c6 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Wed, 11 Sep 2019 10:37:19 +0800 Subject: [PATCH 14/31] recoder --- .../com/qiniu/android/common/AutoZone.java | 6 -- .../com/qiniu/android/common/ZoneInfo.java | 2 - .../com/qiniu/android/http/DnsPrefetcher.java | 99 +++++-------------- .../qiniu/android/storage/UploadManager.java | 10 +- .../qiniu/android/utils/AndroidNetwork.java | 34 ++++++- 5 files changed, 59 insertions(+), 92 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/common/AutoZone.java b/library/src/main/java/com/qiniu/android/common/AutoZone.java index 71cc82a04..267fb2cf2 100644 --- a/library/src/main/java/com/qiniu/android/common/AutoZone.java +++ b/library/src/main/java/com/qiniu/android/common/AutoZone.java @@ -4,7 +4,6 @@ import com.qiniu.android.http.CompletionHandler; import com.qiniu.android.http.ResponseInfo; import com.qiniu.android.storage.UpToken; -import com.qiniu.android.utils.StringMap; import com.qiniu.android.utils.UrlSafeBase64; import org.json.JSONException; @@ -25,7 +24,6 @@ public final class AutoZone extends Zone { private final String ucServer; private Map zones = new ConcurrentHashMap<>(); private Client client = new Client(); - private ZoneInfo zoneInfo; /** * default useHttps to req autoZone @@ -83,7 +81,6 @@ void preQueryIndex(final ZoneIndex index, final QueryHandler complete) { return; } ZoneInfo info = zones.get(index); - zoneInfo = info; if (info != null) { complete.onSuccess(); return; @@ -95,7 +92,6 @@ public void complete(ResponseInfo info, JSONObject response) { if (info.isOK() && response != null) { try { ZoneInfo info2 = ZoneInfo.buildFromJson(response); - zoneInfo = info2; zones.put(index, info2); complete.onSuccess(); return; @@ -116,13 +112,11 @@ boolean preQueryIndex(final ZoneIndex index) { if (index != null) { ZoneInfo info = zones.get(index); if (info != null) { - zoneInfo = info; success = true; } else { try { ResponseInfo responseInfo = getZoneJsonSync(index); ZoneInfo info2 = ZoneInfo.buildFromJson(responseInfo.response); - zoneInfo = info2; zones.put(index, info2); success = true; } catch (JSONException e) { diff --git a/library/src/main/java/com/qiniu/android/common/ZoneInfo.java b/library/src/main/java/com/qiniu/android/common/ZoneInfo.java index d9b6959aa..2e2fd72c3 100644 --- a/library/src/main/java/com/qiniu/android/common/ZoneInfo.java +++ b/library/src/main/java/com/qiniu/android/common/ZoneInfo.java @@ -1,7 +1,5 @@ package com.qiniu.android.common; -import android.util.Log; - import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; diff --git a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java index 3f6e37134..77f9a0498 100644 --- a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java +++ b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java @@ -3,9 +3,7 @@ import com.qiniu.android.collect.Config; import com.qiniu.android.common.Constants; import com.qiniu.android.common.FixedZone; -import com.qiniu.android.common.Zone; import com.qiniu.android.common.ZoneInfo; -import com.qiniu.android.storage.UpToken; import com.qiniu.android.utils.UrlSafeBase64; import org.json.JSONException; @@ -19,7 +17,6 @@ import java.util.List; /** - * *

* Created by jemy on 2019/8/20. */ @@ -27,9 +24,9 @@ public class DnsPrefetcher { public static DnsPrefetcher dnsPrefetcher = null; - private String token; + private static String token; - private HashMap> mConcurrentHashMap = new HashMap>(); + private static HashMap> mConcurrentHashMap = new HashMap>(); private List mHosts = new ArrayList(); @@ -37,8 +34,15 @@ private DnsPrefetcher() { } - public void setConcurrentHashMap(HashMap> mConcurrentHashMap) { - this.mConcurrentHashMap = mConcurrentHashMap; + public static DnsPrefetcher getDnsPrefetcher() { + if (dnsPrefetcher == null) { + synchronized (dnsPrefetcher) { + if (dnsPrefetcher == null) { + dnsPrefetcher = new DnsPrefetcher(); + } + } + } + return dnsPrefetcher; } public DnsPrefetcher init(String token) throws UnknownHostException { @@ -48,6 +52,9 @@ public DnsPrefetcher init(String token) throws UnknownHostException { return this; } + public void setConcurrentHashMap(HashMap> mConcurrentHashMap) { + this.mConcurrentHashMap = mConcurrentHashMap; + } //use for test public List getHosts() { @@ -69,23 +76,9 @@ public void setToken(String token) { } public List getInetAddressByHost(String host) { - if (mConcurrentHashMap != null) { - return mConcurrentHashMap.get(host); - } - return null; + return mConcurrentHashMap.get(host); } - public static DnsPrefetcher getDnsPrefetcher() { - if (dnsPrefetcher == null) { - dnsPrefetcher = new DnsPrefetcher(); - synchronized (dnsPrefetcher) { - if (dnsPrefetcher == null) { - dnsPrefetcher = new DnsPrefetcher(); - } - } - } - return dnsPrefetcher; - } public void preHosts() { HashSet set = new HashSet(); @@ -110,10 +103,16 @@ public void preHosts() { } - public void preFetch() throws UnknownHostException { + public void preFetch() { for (int i = 0; i < mHosts.size(); i++) { - List inetAddresses = getDnsBySystem().lookup(mHosts.get(i)); - mConcurrentHashMap.put(mHosts.get(i), inetAddresses); + List inetAddresses = null; + try { + inetAddresses = getDnsBySystem().lookup(mHosts.get(i)); + mConcurrentHashMap.put(mHosts.get(i), inetAddresses); + } catch (UnknownHostException e) { + e.printStackTrace(); + } + } } @@ -170,9 +169,6 @@ public List getLocalZone() { * query host sync */ public ZoneInfo getPreQueryZone() { - if (token == null) { - return null; - } DnsPrefetcher.ZoneIndex index = DnsPrefetcher.ZoneIndex.getFromToken(token); ZoneInfo zoneInfo = preQueryIndex(index); return zoneInfo; @@ -185,6 +181,7 @@ ZoneInfo preQueryIndex(DnsPrefetcher.ZoneIndex index) { info = ZoneInfo.buildFromJson(responseInfo.response); } catch (JSONException e) { e.printStackTrace(); + return null; } return info; } @@ -196,52 +193,6 @@ ResponseInfo getZoneJsonSync(DnsPrefetcher.ZoneIndex index) { } - /** - * query host async - */ - public void getPreQueryZone(String token, final Zone.QueryHandler complete) { - DnsPrefetcher.ZoneIndex index = DnsPrefetcher.ZoneIndex.getFromToken(token); - - preQueryIndex(index, new CompletionHandler() { - @Override - public void complete(ResponseInfo info, JSONObject response) { - if (info.isOK() && response != null) { - try { - ZoneInfo zoneInfo = ZoneInfo.buildFromJson(response); - for (int i = 0; i < zoneInfo.upDomainsList.size(); i++) { - mHosts.add(zoneInfo.upDomainsList.get(i)); - } - complete.onSuccess(); - return; - } catch (JSONException e) { - e.printStackTrace(); - complete.onFailure(ResponseInfo.NetworkError); - return; - } - } - } - }); - } - - ZoneInfo preQueryIndex(DnsPrefetcher.ZoneIndex index, CompletionHandler handler) { - ZoneInfo info = null; - try { - ResponseInfo responseInfo = getZoneJsonAsync(index, handler); - info = ZoneInfo.buildFromJson(responseInfo.response); - } catch (JSONException e) { - e.printStackTrace(); - } - return info; - } - - private ResponseInfo getZoneJsonAsync(ZoneIndex index, CompletionHandler handler) { - Client client = new Client(); - String address = "http://" + Config.preQueryHost + "/v2/query?ak=" + index.accessKey + "&bucket=" + index.bucket; - client.asyncGet(address, null, UpToken.NULL, handler); - return null; - } - - static class ZoneIndex { final String accessKey; final String bucket; diff --git a/library/src/main/java/com/qiniu/android/storage/UploadManager.java b/library/src/main/java/com/qiniu/android/storage/UploadManager.java index 736942d9d..4fb231e06 100644 --- a/library/src/main/java/com/qiniu/android/storage/UploadManager.java +++ b/library/src/main/java/com/qiniu/android/storage/UploadManager.java @@ -1,12 +1,8 @@ package com.qiniu.android.storage; -import android.content.SharedPreferences; -import android.util.Log; - import com.qiniu.android.collect.Config; import com.qiniu.android.collect.UploadInfoCollector; import com.qiniu.android.common.Zone; -import com.qiniu.android.common.ZoneInfo; import com.qiniu.android.http.Client; import com.qiniu.android.http.DnsPrefetcher; import com.qiniu.android.http.ResponseInfo; @@ -15,14 +11,12 @@ import com.qiniu.android.utils.AsyncRun; import com.qiniu.android.utils.StringUtils; -import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import java.io.File; import java.io.IOException; import java.net.InetAddress; -import java.net.UnknownHostException; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -360,7 +354,7 @@ public boolean checkRePrefetchDns(String token) { recorder = new FileRecorder(Config.dnscacheDir); } catch (IOException e) { e.printStackTrace(); - return false; + return true; } byte[] data = recorder.get("lastcache"); if (data == null) { @@ -392,6 +386,7 @@ public boolean checkRePrefetchDns(String token) { /** * start preFetchDns: Time-consuming operation, in a thread + * * @param token */ public void startPrefetchDns(String token) { @@ -418,7 +413,6 @@ public void startPrefetchDns(String token) { } /** - * * @param recorder * @return */ diff --git a/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java b/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java index e2ca717ca..9959c050e 100644 --- a/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java +++ b/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java @@ -3,7 +3,6 @@ import android.content.Context; import android.net.ConnectivityManager; import android.net.NetworkInfo; -import android.util.Log; import java.net.Inet6Address; import java.net.InetAddress; @@ -32,11 +31,14 @@ public static boolean isNetWorkReady() { /** * 获取ip地址 + * 使用DNS解析某地址时，可能会同时返回IPv4和IPv6的地址。 + * 如果同时拥有IPv4和IPv6的地址，是会默认优先上报IPv6的地址 * * @return */ public static String getHostIP() { String hostIp = null; + String ipv6 = null; try { Enumeration nis = NetworkInterface.getNetworkInterfaces(); InetAddress ia = null; @@ -46,7 +48,11 @@ public static String getHostIP() { while (ias.hasMoreElements()) { ia = ias.nextElement(); if (ia instanceof Inet6Address) { - continue;// skip ipv6 + ipv6 = checkIpv6(ia.getHostAddress()); + if (ipv6 != null) { + return ipv6; + } + continue; } String ip = ia.getHostAddress(); if (!"127.0.0.1".equals(ip)) { @@ -60,4 +66,28 @@ public static String getHostIP() { } return hostIp; } + + /** + * first segment contains "fe" or "fc": https://blog.csdn.net/fdl19881/article/details/7091138 + * + * @param ipv6 + * @return + */ + private static String checkIpv6(String ipv6) { + if (ipv6 != null && ipv6.contains("%")) { + String[] split = ipv6.split("%"); + String s1 = split[0]; + if (s1 != null && s1.contains(":")) { + String[] split1 = s1.split(":"); + if (split1.length == 6 || split1.length == 8) { + if (split1[0].contains("fe") || split1[0].contains("fc")) { + return null; + } else { + return s1; + } + } + } + } + return null; + } } From 43c110d49eb2f5ba8f1d84306e0c86c41363e35d Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Sat, 14 Sep 2019 11:41:59 +0800 Subject: [PATCH 15/31] review and code protect --- .../java/com/qiniu/android/DnsApiTest.java | 67 ++++++++++++--- .../com/qiniu/android/collect/Config.java | 4 +- .../java/com/qiniu/android/http/Client.java | 9 +- .../com/qiniu/android/http/DnsPrefetcher.java | 82 +++++++++++-------- .../qiniu/android/storage/Configuration.java | 7 +- .../qiniu/android/storage/UploadManager.java | 39 +++++---- 6 files changed, 133 insertions(+), 75 deletions(-) diff --git a/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java b/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java index 3b9e5d3d0..d72de5f9f 100644 --- a/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java +++ b/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java @@ -1,31 +1,30 @@ package com.qiniu.android; -import android.graphics.LinearGradient; -import android.test.AndroidTestCase; + import android.test.InstrumentationTestCase; import android.util.Log; import com.qiniu.android.collect.Config; import com.qiniu.android.common.ZoneInfo; import com.qiniu.android.http.DnsPrefetcher; +import com.qiniu.android.http.ResponseInfo; +import com.qiniu.android.storage.Configuration; import com.qiniu.android.storage.Recorder; +import com.qiniu.android.storage.UpCompletionHandler; +import com.qiniu.android.storage.UpProgressHandler; import com.qiniu.android.storage.UploadManager; +import com.qiniu.android.storage.UploadOptions; import com.qiniu.android.storage.persistent.FileRecorder; import com.qiniu.android.utils.AndroidNetwork; import com.qiniu.android.utils.StringUtils; -import org.json.JSONArray; -import org.json.JSONException; import org.json.JSONObject; +import java.io.File; import java.io.IOException; -import java.io.Serializable; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; +import java.util.Hashtable; import java.util.List; /** @@ -134,7 +133,7 @@ public void testDnsPreAndcache() { } //预取或者recover success List list = DnsPrefetcher.getDnsPrefetcher().getHosts(); - HashMap> map = DnsPrefetcher.getDnsPrefetcher().getConcurrentHashMap(); + Hashtable> map = DnsPrefetcher.getDnsPrefetcher().getConcurrentHashMap(); Log.e("qiniutest: ", "list size: " + list.size()); for (String s : list) { Log.e("qiniutest: ", "uphost: " + s); @@ -159,7 +158,7 @@ public void testRecoverCache() { new UploadManager().recoverDnsCache(recorder); - HashMap> map1 = DnsPrefetcher.getDnsPrefetcher().getConcurrentHashMap(); + Hashtable> map1 = DnsPrefetcher.getDnsPrefetcher().getConcurrentHashMap(); List list = DnsPrefetcher.getDnsPrefetcher().getHosts(); for (String s : list) { Log.e("qiniutest: ", "uphost for cache: " + s); @@ -171,5 +170,51 @@ public void testRecoverCache() { } } + int time = 0; + final Object lock = new Object(); + + public void testAtomic() { + final int size = 6 * 1024; + for (int i = 0; i < 3; i++) { + new Thread(new Runnable() { + @Override + public void run() { + try { + Configuration config = new Configuration.Builder().build(); + final UploadManager uploadManager = new UploadManager(config, 3); + final String expectKey = "r=" + size + "k"; + final File f; + f = TempFile.createFile(size); + final UploadOptions uploadoption = new UploadOptions(null, null, false, new UpProgressHandler() { + public void progress(String key, double percent) { + Log.e("qiniutest", percent + ""); + } + }, null); + + uploadManager.put(f, expectKey, TestConfig.token_z0, new UpCompletionHandler() { + public void complete(String k, ResponseInfo rinfo, JSONObject response) { + Log.e("qiniutest", k + rinfo); + time += 1; + if (time == 3) { + lock.notify(); + } + } + }, uploadoption); + + + } catch (IOException e) { + e.printStackTrace(); + } + } + }).start(); + } + synchronized (lock) { + try { + lock.wait(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } } diff --git a/library/src/main/java/com/qiniu/android/collect/Config.java b/library/src/main/java/com/qiniu/android/collect/Config.java index 7793a193d..8917a8405 100644 --- a/library/src/main/java/com/qiniu/android/collect/Config.java +++ b/library/src/main/java/com/qiniu/android/collect/Config.java @@ -77,10 +77,12 @@ public final class Config { public static String dnscacheDir = "/sdcard/dnschache"; /** - * + * preQuery host */ public static String preQueryHost = "uc.qbox.me"; + public static int rePreHost = 2; + /** * 当网络切换到 wifi 下，切换到此设置 */ diff --git a/library/src/main/java/com/qiniu/android/http/Client.java b/library/src/main/java/com/qiniu/android/http/Client.java index d5454b10b..b9ce4a818 100644 --- a/library/src/main/java/com/qiniu/android/http/Client.java +++ b/library/src/main/java/com/qiniu/android/http/Client.java @@ -1,13 +1,8 @@ package com.qiniu.android.http; -import android.util.Log; - -import com.qiniu.android.collect.Config; import com.qiniu.android.common.Constants; -import com.qiniu.android.storage.Recorder; import com.qiniu.android.storage.UpCancellationSignal; import com.qiniu.android.storage.UpToken; -import com.qiniu.android.storage.persistent.FileRecorder; import com.qiniu.android.utils.AsyncRun; import com.qiniu.android.utils.StringMap; import com.qiniu.android.utils.StringUtils; @@ -18,8 +13,6 @@ import java.net.InetAddress; import java.net.SocketTimeoutException; import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.concurrent.TimeUnit; @@ -444,7 +437,7 @@ public void accept(String key, Object value) { } } - public static class ResponseTag { + private static class ResponseTag { public String ip = ""; public long duration = -1; } diff --git a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java index 77f9a0498..7853194d9 100644 --- a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java +++ b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java @@ -12,8 +12,8 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; +import java.util.Hashtable; import java.util.List; /** @@ -26,9 +26,8 @@ public class DnsPrefetcher { public static DnsPrefetcher dnsPrefetcher = null; private static String token; - private static HashMap> mConcurrentHashMap = new HashMap>(); - private List mHosts = new ArrayList(); - + private static Hashtable> mConcurrentHashMap = new Hashtable>(); + private static List mHosts = new ArrayList(); private DnsPrefetcher() { @@ -36,7 +35,7 @@ private DnsPrefetcher() { public static DnsPrefetcher getDnsPrefetcher() { if (dnsPrefetcher == null) { - synchronized (dnsPrefetcher) { + synchronized (DnsPrefetcher.class) { if (dnsPrefetcher == null) { dnsPrefetcher = new DnsPrefetcher(); } @@ -52,7 +51,7 @@ public DnsPrefetcher init(String token) throws UnknownHostException { return this; } - public void setConcurrentHashMap(HashMap> mConcurrentHashMap) { + public void setConcurrentHashMap(Hashtable> mConcurrentHashMap) { this.mConcurrentHashMap = mConcurrentHashMap; } @@ -66,7 +65,7 @@ public void setHosts(List mHosts) { } //use for test - public HashMap> getConcurrentHashMap() { + public Hashtable> getConcurrentHashMap() { return this.mConcurrentHashMap; } @@ -79,8 +78,7 @@ public List getInetAddressByHost(String host) { return mConcurrentHashMap.get(host); } - - public void preHosts() { + private void preHosts() { HashSet set = new HashSet(); //preQuery sync @@ -99,23 +97,58 @@ public void preHosts() { mHosts.add(zone.upDomainsList.get(i)); } } - mHosts.add(Config.preQueryHost); + if (set.add(Config.preQueryHost)) + mHosts.add(Config.preQueryHost); } - public void preFetch() { + private void preFetch() { + List rePreHosts = new ArrayList(); for (int i = 0; i < mHosts.size(); i++) { List inetAddresses = null; try { - inetAddresses = getDnsBySystem().lookup(mHosts.get(i)); + inetAddresses = okhttp3.Dns.SYSTEM.lookup(mHosts.get(i)); mConcurrentHashMap.put(mHosts.get(i), inetAddresses); } catch (UnknownHostException e) { e.printStackTrace(); + rePreHosts.add(mHosts.get(i)); } + } + rePreFetch(rePreHosts, null); + } + /** + * 对hosts预取失败对进行重新预取，deafult retryNum = 2 + * + * @param rePreHosts 用于重试的hosts + * @param customeDns 是否自定义dns + */ + private void rePreFetch(List rePreHosts, Dns customeDns) { + for (int i = 0; i < rePreHosts.size(); i++) { + int rePreNum = 0; + while (rePreNum < Config.rePreHost) { + rePreNum += 1; + if (rePreFetch(rePreHosts.get(i), customeDns)) + break; + } } } + private boolean rePreFetch(String host, Dns customeDns) { + List inetAddresses = null; + try { + if (customeDns == null) { + inetAddresses = okhttp3.Dns.SYSTEM.lookup(host); + } else { + inetAddresses = customeDns.lookup(host); + } + mConcurrentHashMap.put(host, inetAddresses); + return true; + } catch (UnknownHostException e) { + e.printStackTrace(); + return false; + } + } /** * 自定义dns预取 @@ -125,37 +158,20 @@ public void preFetch() { * @throws UnknownHostException */ public void dnsPreByCustom(Dns dns) { + List rePreHosts = new ArrayList(); for (int i = 0; i < mHosts.size(); i++) { List inetAddresses = null; try { inetAddresses = dns.lookup(mHosts.get(i)); + mConcurrentHashMap.put(mHosts.get(i), inetAddresses); } catch (UnknownHostException e) { e.printStackTrace(); + rePreHosts.add(mHosts.get(i)); } - mConcurrentHashMap.put(mHosts.get(i), inetAddresses); } + rePreFetch(rePreHosts, dns); } - - /** - * 系统DNS解析预取 - * - * @return - * @throws UnknownHostException - */ - public Dns getDnsBySystem() throws UnknownHostException { - return new SystemDns(); - } - - - class SystemDns implements Dns { - @Override - public List lookup(String hostname) throws UnknownHostException { - return okhttp3.Dns.SYSTEM.lookup(hostname); - } - } - - /** * look local host */ diff --git a/library/src/main/java/com/qiniu/android/storage/Configuration.java b/library/src/main/java/com/qiniu/android/storage/Configuration.java index efbac2651..1c65c2496 100644 --- a/library/src/main/java/com/qiniu/android/storage/Configuration.java +++ b/library/src/main/java/com/qiniu/android/storage/Configuration.java @@ -91,11 +91,8 @@ private Configuration(Builder builder) { urlConverter = builder.urlConverter; AutoZone autoZone = null; - if (builder.useHttps) { - autoZone = new AutoZone(); - } else { - autoZone = new AutoZone(false); - } + + autoZone = new AutoZone(builder.useHttps); zone = builder.zone == null ? autoZone : builder.zone; dns = builder.dns; diff --git a/library/src/main/java/com/qiniu/android/storage/UploadManager.java b/library/src/main/java/com/qiniu/android/storage/UploadManager.java index 4fb231e06..50a4821b1 100644 --- a/library/src/main/java/com/qiniu/android/storage/UploadManager.java +++ b/library/src/main/java/com/qiniu/android/storage/UploadManager.java @@ -18,10 +18,11 @@ import java.io.IOException; import java.net.InetAddress; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Hashtable; import java.util.Iterator; import java.util.List; import java.util.Locale; +import java.util.concurrent.atomic.AtomicBoolean; import static java.lang.String.format; @@ -36,6 +37,7 @@ public final class UploadManager { private final Client client; private int multithreads = 1; private static int DEF_THREAD_NUM = 3; + static AtomicBoolean atomic = new AtomicBoolean(false); /** * default 3 Threads @@ -150,14 +152,15 @@ public void put(final byte[] data, final String key, final String token, } if (checkRePrefetchDns(token)) { - new Thread(new Runnable() { - @Override - public void run() { - startPrefetchDns(token); - } - }).start(); + if (atomic.compareAndSet(false, true)) { + new Thread(new Runnable() { + @Override + public void run() { + startPrefetchDns(token); + } + }).start(); + } } - Zone z = config.zone; z.preQuery(token, new Zone.QueryHandler() { @Override @@ -190,6 +193,7 @@ public void put(String filePath, String key, String token, UpCompletionHandler c put(new File(filePath), key, token, completionHandler, options); } + /** * 上传文件 * @@ -207,14 +211,15 @@ public void put(final File file, final String key, final String token, final UpC } if (checkRePrefetchDns(token)) { - new Thread(new Runnable() { - @Override - public void run() { - startPrefetchDns(token); - } - }).start(); + if (atomic.compareAndSet(false, true)) { + new Thread(new Runnable() { + @Override + public void run() { + startPrefetchDns(token); + } + }).start(); + } } - Zone z = config.zone; z.preQuery(token, new Zone.QueryHandler() { @Override @@ -405,7 +410,7 @@ public void startPrefetchDns(String token) { if (config.dns != null) { DnsPrefetcher.getDnsPrefetcher().dnsPreByCustom(config.dns); } - HashMap> concurrentHashMap = dnsPrefetcher.getConcurrentHashMap(); + Hashtable> concurrentHashMap = dnsPrefetcher.getConcurrentHashMap(); byte[] dnscache = StringUtils.toByteArray(concurrentHashMap); recorder.set("lastcache", data.getBytes()); @@ -417,7 +422,7 @@ public void startPrefetchDns(String token) { * @return */ public boolean recoverDnsCache(Recorder recorder) { - HashMap> concurrentHashMap = (HashMap>) StringUtils.toObject(recorder.get("dnscache")); + Hashtable> concurrentHashMap = (Hashtable>) StringUtils.toObject(recorder.get("dnscache")); if (concurrentHashMap == null) { return true; } From 172e671775e2613d3ab4f0eba28918227e0efe7b Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Mon, 16 Sep 2019 14:53:37 +0800 Subject: [PATCH 16/31] concurrentHashMap --- .idea/misc.xml | 2 +- .../androidTest/java/com/qiniu/android/DnsApiTest.java | 6 +++--- .../main/java/com/qiniu/android/http/DnsPrefetcher.java | 8 ++++---- .../java/com/qiniu/android/storage/UploadManager.java | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index e0d5b93ff..b0c7b20c8 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -29,7 +29,7 @@ - + diff --git a/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java b/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java index d72de5f9f..e81fc5a9f 100644 --- a/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java +++ b/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java @@ -24,8 +24,8 @@ import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.Hashtable; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; /** * Created by jemy on 2019/8/20. @@ -133,7 +133,7 @@ public void testDnsPreAndcache() { } //预取或者recover success List list = DnsPrefetcher.getDnsPrefetcher().getHosts(); - Hashtable> map = DnsPrefetcher.getDnsPrefetcher().getConcurrentHashMap(); + ConcurrentHashMap> map = DnsPrefetcher.getDnsPrefetcher().getConcurrentHashMap(); Log.e("qiniutest: ", "list size: " + list.size()); for (String s : list) { Log.e("qiniutest: ", "uphost: " + s); @@ -158,7 +158,7 @@ public void testRecoverCache() { new UploadManager().recoverDnsCache(recorder); - Hashtable> map1 = DnsPrefetcher.getDnsPrefetcher().getConcurrentHashMap(); + ConcurrentHashMap> map1 = DnsPrefetcher.getDnsPrefetcher().getConcurrentHashMap(); List list = DnsPrefetcher.getDnsPrefetcher().getHosts(); for (String s : list) { Log.e("qiniutest: ", "uphost for cache: " + s); diff --git a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java index 7853194d9..ab2bccd4b 100644 --- a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java +++ b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java @@ -13,8 +13,8 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.HashSet; -import java.util.Hashtable; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; /** *

@@ -26,7 +26,7 @@ public class DnsPrefetcher { public static DnsPrefetcher dnsPrefetcher = null; private static String token; - private static Hashtable> mConcurrentHashMap = new Hashtable>(); + private static ConcurrentHashMap> mConcurrentHashMap = new ConcurrentHashMap>(); private static List mHosts = new ArrayList(); private DnsPrefetcher() { @@ -51,7 +51,7 @@ public DnsPrefetcher init(String token) throws UnknownHostException { return this; } - public void setConcurrentHashMap(Hashtable> mConcurrentHashMap) { + public void setConcurrentHashMap(ConcurrentHashMap> mConcurrentHashMap) { this.mConcurrentHashMap = mConcurrentHashMap; } @@ -65,7 +65,7 @@ public void setHosts(List mHosts) { } //use for test - public Hashtable> getConcurrentHashMap() { + public ConcurrentHashMap> getConcurrentHashMap() { return this.mConcurrentHashMap; } diff --git a/library/src/main/java/com/qiniu/android/storage/UploadManager.java b/library/src/main/java/com/qiniu/android/storage/UploadManager.java index 50a4821b1..1efeb82b8 100644 --- a/library/src/main/java/com/qiniu/android/storage/UploadManager.java +++ b/library/src/main/java/com/qiniu/android/storage/UploadManager.java @@ -18,10 +18,10 @@ import java.io.IOException; import java.net.InetAddress; import java.util.ArrayList; -import java.util.Hashtable; import java.util.Iterator; import java.util.List; import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import static java.lang.String.format; @@ -410,7 +410,7 @@ public void startPrefetchDns(String token) { if (config.dns != null) { DnsPrefetcher.getDnsPrefetcher().dnsPreByCustom(config.dns); } - Hashtable> concurrentHashMap = dnsPrefetcher.getConcurrentHashMap(); + ConcurrentHashMap> concurrentHashMap = dnsPrefetcher.getConcurrentHashMap(); byte[] dnscache = StringUtils.toByteArray(concurrentHashMap); recorder.set("lastcache", data.getBytes()); @@ -422,7 +422,7 @@ public void startPrefetchDns(String token) { * @return */ public boolean recoverDnsCache(Recorder recorder) { - Hashtable> concurrentHashMap = (Hashtable>) StringUtils.toObject(recorder.get("dnscache")); + ConcurrentHashMap> concurrentHashMap = (ConcurrentHashMap>) StringUtils.toObject(recorder.get("dnscache")); if (concurrentHashMap == null) { return true; } From 5ece256f389966297963020fdf6094bf70bbc9df Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Mon, 16 Sep 2019 20:29:04 +0800 Subject: [PATCH 17/31] change code style --- .../com/qiniu/android/http/DnsPrefetcher.java | 106 ++++++++++++++++++ .../qiniu/android/storage/UploadManager.java | 104 +---------------- 2 files changed, 110 insertions(+), 100 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java index ab2bccd4b..f76631a6d 100644 --- a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java +++ b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java @@ -4,18 +4,28 @@ import com.qiniu.android.common.Constants; import com.qiniu.android.common.FixedZone; import com.qiniu.android.common.ZoneInfo; +import com.qiniu.android.storage.Configuration; +import com.qiniu.android.storage.Recorder; +import com.qiniu.android.storage.persistent.FileRecorder; +import com.qiniu.android.utils.AndroidNetwork; +import com.qiniu.android.utils.StringUtils; import com.qiniu.android.utils.UrlSafeBase64; import org.json.JSONException; import org.json.JSONObject; +import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.concurrent.ConcurrentHashMap; +import static java.lang.String.format; + /** *

* Created by jemy on 2019/8/20. @@ -243,4 +253,100 @@ public boolean equals(Object obj) { && ((DnsPrefetcher.ZoneIndex) obj).accessKey.equals(accessKey) && ((DnsPrefetcher.ZoneIndex) obj).bucket.equals(bucket); } } + + /** + *

+ * ip changed, the network has changed + * ak:scope变化，prequery（v2）自动获取域名接口发生变化，存储区域可能变化 + * cacheTime>config.cacheTime（默认24H） + *

+ * + * @return true:重新预期并缓存, false:不需要重新预取和缓存 + */ + public static boolean checkRePrefetchDns(String token, Configuration config) { + Recorder recorder = null; + try { + recorder = new FileRecorder(Config.dnscacheDir); + } catch (IOException e) { + e.printStackTrace(); + return true; + } + byte[] data = recorder.get("lastcache"); + if (data == null) { + return true; + } + String jsonStr = new String(data); + JSONObject obj; + try { + obj = new JSONObject(jsonStr); + } catch (JSONException e) { + e.printStackTrace(); + return true; + } + String time = obj.optString("time"); + String ip = obj.optString("ip"); + String ak = obj.optString("ak"); + + String currentTime = String.valueOf(System.currentTimeMillis()); + String localip = AndroidNetwork.getHostIP(); + String akAndScope = StringUtils.getAkAndScope(token); + + long cacheTime = (Long.parseLong(currentTime) - Long.parseLong(time)) / 1000; + if (!localip.equals(ip) || cacheTime > config.dnsCacheTimeMs || !akAndScope.equals(ak)) { + return true; + } + + return recoverDnsCache(recorder); + } + + /** + * start preFetchDns: Time-consuming operation, in a thread + * + * @param token + */ + public static void startPrefetchDns(String token, Configuration config) { + String currentTime = String.valueOf(System.currentTimeMillis()); + String localip = AndroidNetwork.getHostIP(); + String akAndScope = StringUtils.getAkAndScope(token); + String data = format(Locale.ENGLISH, "{\"time\":%s,\"ip\":%s,\"ak\":%s}", currentTime, localip, akAndScope); + Recorder recorder = null; + DnsPrefetcher dnsPrefetcher = null; + try { + recorder = new FileRecorder(Config.dnscacheDir); + dnsPrefetcher = DnsPrefetcher.getDnsPrefetcher().init(token); + } catch (IOException e) { + e.printStackTrace(); + } + if (config.dns != null) { + DnsPrefetcher.getDnsPrefetcher().dnsPreByCustom(config.dns); + } + ConcurrentHashMap> concurrentHashMap = dnsPrefetcher.getConcurrentHashMap(); + byte[] dnscache = StringUtils.toByteArray(concurrentHashMap); + + recorder.set("lastcache", data.getBytes()); + recorder.set("dnscache", dnscache); + } + + /** + * @param recorder + * @return + */ + public static boolean recoverDnsCache(Recorder recorder) { + ConcurrentHashMap> concurrentHashMap = (ConcurrentHashMap>) StringUtils.toObject(recorder.get("dnscache")); + if (concurrentHashMap == null) { + return true; + } + DnsPrefetcher.getDnsPrefetcher().setConcurrentHashMap(concurrentHashMap); + + ArrayList list = new ArrayList(); + Iterator iter = concurrentHashMap.keySet().iterator(); + while (iter.hasNext()) { + String tmpkey = (String) iter.next(); + if (tmpkey == null || tmpkey.length() == 0) + continue; + list.add(tmpkey); + } + DnsPrefetcher.getDnsPrefetcher().setHosts(list); + return false; + } } diff --git a/library/src/main/java/com/qiniu/android/storage/UploadManager.java b/library/src/main/java/com/qiniu/android/storage/UploadManager.java index 1efeb82b8..1334c5f68 100644 --- a/library/src/main/java/com/qiniu/android/storage/UploadManager.java +++ b/library/src/main/java/com/qiniu/android/storage/UploadManager.java @@ -151,12 +151,12 @@ public void put(final byte[] data, final String key, final String token, return; } - if (checkRePrefetchDns(token)) { + if (DnsPrefetcher.checkRePrefetchDns(token, config)) { if (atomic.compareAndSet(false, true)) { new Thread(new Runnable() { @Override public void run() { - startPrefetchDns(token); + DnsPrefetcher.startPrefetchDns(token, config); } }).start(); } @@ -210,12 +210,12 @@ public void put(final File file, final String key, final String token, final UpC return; } - if (checkRePrefetchDns(token)) { + if (DnsPrefetcher.checkRePrefetchDns(token, config)) { if (atomic.compareAndSet(false, true)) { new Thread(new Runnable() { @Override public void run() { - startPrefetchDns(token); + DnsPrefetcher.startPrefetchDns(token, config); } }).start(); } @@ -344,100 +344,4 @@ public void run() { } - /** - *

- * ip changed, the network has changed - * ak:scope变化，prequery（v2）自动获取域名接口发生变化，存储区域可能变化 - * cacheTime>config.cacheTime（默认24H） - *

- * - * @return true:重新预期并缓存, false:不需要重新预取和缓存 - */ - public boolean checkRePrefetchDns(String token) { - Recorder recorder = null; - try { - recorder = new FileRecorder(Config.dnscacheDir); - } catch (IOException e) { - e.printStackTrace(); - return true; - } - byte[] data = recorder.get("lastcache"); - if (data == null) { - return true; - } - String jsonStr = new String(data); - JSONObject obj; - try { - obj = new JSONObject(jsonStr); - } catch (JSONException e) { - e.printStackTrace(); - return true; - } - String time = obj.optString("time"); - String ip = obj.optString("ip"); - String ak = obj.optString("ak"); - - String currentTime = String.valueOf(System.currentTimeMillis()); - String localip = AndroidNetwork.getHostIP(); - String akAndScope = StringUtils.getAkAndScope(token); - - long cacheTime = (Long.parseLong(currentTime) - Long.parseLong(time)) / 1000; - if (!localip.equals(ip) || cacheTime > config.dnsCacheTimeMs || !akAndScope.equals(ak)) { - return true; - } - - return recoverDnsCache(recorder); - } - - /** - * start preFetchDns: Time-consuming operation, in a thread - * - * @param token - */ - public void startPrefetchDns(String token) { - String currentTime = String.valueOf(System.currentTimeMillis()); - String localip = AndroidNetwork.getHostIP(); - String akAndScope = StringUtils.getAkAndScope(token); - String data = format(Locale.ENGLISH, "{\"time\":%s,\"ip\":%s,\"ak\":%s}", currentTime, localip, akAndScope); - Recorder recorder = null; - DnsPrefetcher dnsPrefetcher = null; - try { - recorder = new FileRecorder(Config.dnscacheDir); - dnsPrefetcher = DnsPrefetcher.getDnsPrefetcher().init(token); - } catch (IOException e) { - e.printStackTrace(); - } - if (config.dns != null) { - DnsPrefetcher.getDnsPrefetcher().dnsPreByCustom(config.dns); - } - ConcurrentHashMap> concurrentHashMap = dnsPrefetcher.getConcurrentHashMap(); - byte[] dnscache = StringUtils.toByteArray(concurrentHashMap); - - recorder.set("lastcache", data.getBytes()); - recorder.set("dnscache", dnscache); - } - - /** - * @param recorder - * @return - */ - public boolean recoverDnsCache(Recorder recorder) { - ConcurrentHashMap> concurrentHashMap = (ConcurrentHashMap>) StringUtils.toObject(recorder.get("dnscache")); - if (concurrentHashMap == null) { - return true; - } - DnsPrefetcher.getDnsPrefetcher().setConcurrentHashMap(concurrentHashMap); - - ArrayList list = new ArrayList(); - Iterator iter = concurrentHashMap.keySet().iterator(); - while (iter.hasNext()) { - String tmpkey = (String) iter.next(); - if (tmpkey == null || tmpkey.length() == 0) - continue; - list.add(tmpkey); - } - DnsPrefetcher.getDnsPrefetcher().setHosts(list); - return false; - } - } From a1b2e9bb5ad380099ac6bd7de7425ef19e0434bd Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Mon, 16 Sep 2019 20:30:02 +0800 Subject: [PATCH 18/31] change code --- .../com/qiniu/android/storage/UploadManager.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/storage/UploadManager.java b/library/src/main/java/com/qiniu/android/storage/UploadManager.java index 1334c5f68..1adc6a3e1 100644 --- a/library/src/main/java/com/qiniu/android/storage/UploadManager.java +++ b/library/src/main/java/com/qiniu/android/storage/UploadManager.java @@ -6,26 +6,14 @@ import com.qiniu.android.http.Client; import com.qiniu.android.http.DnsPrefetcher; import com.qiniu.android.http.ResponseInfo; -import com.qiniu.android.storage.persistent.FileRecorder; -import com.qiniu.android.utils.AndroidNetwork; import com.qiniu.android.utils.AsyncRun; import com.qiniu.android.utils.StringUtils; -import org.json.JSONException; import org.json.JSONObject; import java.io.File; -import java.io.IOException; -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; -import static java.lang.String.format; - /** * 七牛文件上传管理器 * 一般默认可以使用这个类的方法来上传数据和文件。会自动检测文件的大小， From eee4645d97857a88393ad43def3e31b9b7dbd421 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Tue, 17 Sep 2019 09:37:28 +0800 Subject: [PATCH 19/31] test --- .../androidTest/java/com/qiniu/android/DnsApiTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java b/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java index e81fc5a9f..37d321370 100644 --- a/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java +++ b/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java @@ -122,11 +122,11 @@ public void testSerializeCache() { } public void testDnsPreAndcache() { - UploadManager up = new UploadManager(); - boolean needPrefetch = up.checkRePrefetchDns(Config.dnscacheDir); + Configuration config = new Configuration.Builder().build(); + boolean needPrefetch = DnsPrefetcher.checkRePrefetchDns(TestConfig.uptoken_prefetch, config); Log.e("qiniutest", "check:" + needPrefetch); if (needPrefetch) { - up.startPrefetchDns(TestConfig.uptoken_prefetch); + DnsPrefetcher.startPrefetchDns(TestConfig.uptoken_prefetch, config); } else { testRecoverCache(); return; @@ -155,7 +155,7 @@ public void testRecoverCache() { } catch (IOException e) { e.printStackTrace(); } - new UploadManager().recoverDnsCache(recorder); + DnsPrefetcher.recoverDnsCache(recorder); ConcurrentHashMap> map1 = DnsPrefetcher.getDnsPrefetcher().getConcurrentHashMap(); From 516a8fa75341c1a70c74c4164350d1182c00fc83 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Tue, 17 Sep 2019 15:21:42 +0800 Subject: [PATCH 20/31] recode --- .idea/misc.xml | 2 +- .../java/com/qiniu/android/DnsApiTest.java | 42 +----- .../java/com/qiniu/android/TestCompany.java | 30 ----- .../qiniu/android/common/AutoZoneTest.java | 3 - .../com/qiniu/android/http/DnsPrefetcher.java | 45 +++---- .../com/qiniu/android/storage/Recorder.java | 2 + .../qiniu/android/storage/UploadManager.java | 8 +- .../storage/persistent/DnsCacheFile.java | 127 ++++++++++++++++++ .../storage/persistent/FileRecorder.java | 7 +- 9 files changed, 167 insertions(+), 99 deletions(-) delete mode 100644 library/src/androidTest/java/com/qiniu/android/TestCompany.java create mode 100644 library/src/main/java/com/qiniu/android/storage/persistent/DnsCacheFile.java diff --git a/.idea/misc.xml b/.idea/misc.xml index b0c7b20c8..e0d5b93ff 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -29,7 +29,7 @@ - + diff --git a/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java b/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java index 37d321370..befa30722 100644 --- a/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java +++ b/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java @@ -14,9 +14,8 @@ import com.qiniu.android.storage.UpProgressHandler; import com.qiniu.android.storage.UploadManager; import com.qiniu.android.storage.UploadOptions; -import com.qiniu.android.storage.persistent.FileRecorder; +import com.qiniu.android.storage.persistent.DnsCacheFile; import com.qiniu.android.utils.AndroidNetwork; -import com.qiniu.android.utils.StringUtils; import org.json.JSONObject; @@ -92,41 +91,12 @@ public void testLocalIp() { Log.e("qiniutest", s); } - - public void testSerializeCache() { - String recordKey = "/sdcard/dnschache"; - try { - Recorder recorder = new FileRecorder(Config.dnscacheDir); - - String s = String.valueOf(System.currentTimeMillis()); - TestCompany company = new TestCompany("qiniu", 8); - byte[] com = StringUtils.toByteArray(company); - Log.e("qiniutest", s); - recorder.set("time", s.getBytes()); - recorder.set("compant", com); - - try { - Thread.sleep(900); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - byte[] time = recorder.get("time"); - byte[] getcom = recorder.get("compant"); - Log.e("qiniutest", new String(time)); - TestCompany company1 = (TestCompany) StringUtils.toObject(getcom); - Log.e("qiniutest", "name: " + company1.getName() + " ,age: " + company1.getAge()); - } catch (IOException e) { - e.printStackTrace(); - } - } - public void testDnsPreAndcache() { Configuration config = new Configuration.Builder().build(); - boolean needPrefetch = DnsPrefetcher.checkRePrefetchDns(TestConfig.uptoken_prefetch, config); + boolean needPrefetch = DnsPrefetcher.checkRePrefetchDns("MP_Ebql_lSsUrDr7WrXn_5vKocQDLvTPCNEFeVmp:5mVFMc75Yy4nWYJ8E5j5ESW51Rs=:eyJzY29wZSI6ImFuZHJvaWR0ZXN0IiwiZGVhZGxpbmUiOjE1Njg3MDcxOTl9", config); Log.e("qiniutest", "check:" + needPrefetch); if (needPrefetch) { - DnsPrefetcher.startPrefetchDns(TestConfig.uptoken_prefetch, config); + DnsPrefetcher.startPrefetchDns("MP_Ebql_lSsUrDr7WrXn_5vKocQDLvTPCNEFeVmp:5mVFMc75Yy4nWYJ8E5j5ESW51Rs=:eyJzY29wZSI6ImFuZHJvaWR0ZXN0IiwiZGVhZGxpbmUiOjE1Njg3MDcxOTl9", config); } else { testRecoverCache(); return; @@ -151,15 +121,17 @@ public void testRecoverCache() { Recorder recorder = null; try { - recorder = new FileRecorder(Config.dnscacheDir); + recorder = new DnsCacheFile(Config.dnscacheDir); } catch (IOException e) { e.printStackTrace(); } - DnsPrefetcher.recoverDnsCache(recorder); + byte[] data = recorder.get(recorder.getFileName()); + DnsPrefetcher.recoverDnsCache(data); ConcurrentHashMap> map1 = DnsPrefetcher.getDnsPrefetcher().getConcurrentHashMap(); List list = DnsPrefetcher.getDnsPrefetcher().getHosts(); + Log.e("qiniutest: ", "size for cache: " + list.size()); for (String s : list) { Log.e("qiniutest: ", "uphost for cache: " + s); List list1 = map1.get(s); diff --git a/library/src/androidTest/java/com/qiniu/android/TestCompany.java b/library/src/androidTest/java/com/qiniu/android/TestCompany.java deleted file mode 100644 index 59c6dcb0c..000000000 --- a/library/src/androidTest/java/com/qiniu/android/TestCompany.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.qiniu.android; - -import java.io.Serializable; - -/** - * Created by jemy on 2019/9/5. - */ - -public class TestCompany implements Serializable { - String name; - int age; - - public TestCompany(String name, int age) { - this.name = name; - this.age = age; - } - - public String getName() { - return name; - } - - public int getAge() { - return age; - } - - @Override - public String toString() { - return super.toString(); - } -} diff --git a/library/src/androidTest/java/com/qiniu/android/common/AutoZoneTest.java b/library/src/androidTest/java/com/qiniu/android/common/AutoZoneTest.java index 58ff5659a..520853c28 100644 --- a/library/src/androidTest/java/com/qiniu/android/common/AutoZoneTest.java +++ b/library/src/androidTest/java/com/qiniu/android/common/AutoZoneTest.java @@ -4,12 +4,9 @@ import android.util.Log; import com.qiniu.android.TestConfig; -import com.qiniu.android.http.DnsPrefetcher; import junit.framework.Assert; -import java.net.UnknownHostException; -import java.util.List; import java.util.concurrent.CountDownLatch; /** diff --git a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java index f76631a6d..20fc4f989 100644 --- a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java +++ b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java @@ -6,7 +6,7 @@ import com.qiniu.android.common.ZoneInfo; import com.qiniu.android.storage.Configuration; import com.qiniu.android.storage.Recorder; -import com.qiniu.android.storage.persistent.FileRecorder; +import com.qiniu.android.storage.persistent.DnsCacheFile; import com.qiniu.android.utils.AndroidNetwork; import com.qiniu.android.utils.StringUtils; import com.qiniu.android.utils.UrlSafeBase64; @@ -266,37 +266,33 @@ public boolean equals(Object obj) { public static boolean checkRePrefetchDns(String token, Configuration config) { Recorder recorder = null; try { - recorder = new FileRecorder(Config.dnscacheDir); + recorder = new DnsCacheFile(Config.dnscacheDir); } catch (IOException e) { e.printStackTrace(); return true; } - byte[] data = recorder.get("lastcache"); - if (data == null) { + String dnscache = recorder.getFileName(); + if (dnscache == null) return true; - } - String jsonStr = new String(data); - JSONObject obj; - try { - obj = new JSONObject(jsonStr); - } catch (JSONException e) { - e.printStackTrace(); + + byte[] data = recorder.get(dnscache); + if (data == null) + return true; + + String[] cacheKey = dnscache.split(":"); + if (cacheKey.length < 3) return true; - } - String time = obj.optString("time"); - String ip = obj.optString("ip"); - String ak = obj.optString("ak"); String currentTime = String.valueOf(System.currentTimeMillis()); String localip = AndroidNetwork.getHostIP(); String akAndScope = StringUtils.getAkAndScope(token); - long cacheTime = (Long.parseLong(currentTime) - Long.parseLong(time)) / 1000; - if (!localip.equals(ip) || cacheTime > config.dnsCacheTimeMs || !akAndScope.equals(ak)) { + long cacheTime = (Long.parseLong(currentTime) - Long.parseLong(cacheKey[0])) / 1000; + if (!localip.equals(cacheKey[1]) || cacheTime > config.dnsCacheTimeMs || !akAndScope.equals(cacheKey[2])) { return true; } - return recoverDnsCache(recorder); + return recoverDnsCache(data); } /** @@ -308,11 +304,11 @@ public static void startPrefetchDns(String token, Configuration config) { String currentTime = String.valueOf(System.currentTimeMillis()); String localip = AndroidNetwork.getHostIP(); String akAndScope = StringUtils.getAkAndScope(token); - String data = format(Locale.ENGLISH, "{\"time\":%s,\"ip\":%s,\"ak\":%s}", currentTime, localip, akAndScope); + String cacheKey = format(Locale.ENGLISH, "\"time:\":%s\"ip:\":%s\"ak\":%s", currentTime, localip, akAndScope); Recorder recorder = null; DnsPrefetcher dnsPrefetcher = null; try { - recorder = new FileRecorder(Config.dnscacheDir); + recorder = new DnsCacheFile(Config.dnscacheDir); dnsPrefetcher = DnsPrefetcher.getDnsPrefetcher().init(token); } catch (IOException e) { e.printStackTrace(); @@ -323,16 +319,15 @@ public static void startPrefetchDns(String token, Configuration config) { ConcurrentHashMap> concurrentHashMap = dnsPrefetcher.getConcurrentHashMap(); byte[] dnscache = StringUtils.toByteArray(concurrentHashMap); - recorder.set("lastcache", data.getBytes()); - recorder.set("dnscache", dnscache); + recorder.set(cacheKey, dnscache); } /** - * @param recorder + * @param data * @return */ - public static boolean recoverDnsCache(Recorder recorder) { - ConcurrentHashMap> concurrentHashMap = (ConcurrentHashMap>) StringUtils.toObject(recorder.get("dnscache")); + public static boolean recoverDnsCache(byte[] data) { + ConcurrentHashMap> concurrentHashMap = (ConcurrentHashMap>) StringUtils.toObject(data); if (concurrentHashMap == null) { return true; } diff --git a/library/src/main/java/com/qiniu/android/storage/Recorder.java b/library/src/main/java/com/qiniu/android/storage/Recorder.java index 493a708cb..9bd218cb2 100644 --- a/library/src/main/java/com/qiniu/android/storage/Recorder.java +++ b/library/src/main/java/com/qiniu/android/storage/Recorder.java @@ -27,4 +27,6 @@ public interface Recorder { * @param key 持久化的键 */ void del(String key); + + String getFileName(); } diff --git a/library/src/main/java/com/qiniu/android/storage/UploadManager.java b/library/src/main/java/com/qiniu/android/storage/UploadManager.java index 1adc6a3e1..ec520675a 100644 --- a/library/src/main/java/com/qiniu/android/storage/UploadManager.java +++ b/library/src/main/java/com/qiniu/android/storage/UploadManager.java @@ -139,8 +139,8 @@ public void put(final byte[] data, final String key, final String token, return; } - if (DnsPrefetcher.checkRePrefetchDns(token, config)) { - if (atomic.compareAndSet(false, true)) { + if (atomic.compareAndSet(false, true)) { + if (DnsPrefetcher.checkRePrefetchDns(token, config)) { new Thread(new Runnable() { @Override public void run() { @@ -198,8 +198,8 @@ public void put(final File file, final String key, final String token, final UpC return; } - if (DnsPrefetcher.checkRePrefetchDns(token, config)) { - if (atomic.compareAndSet(false, true)) { + if (atomic.compareAndSet(false, true)) { + if (DnsPrefetcher.checkRePrefetchDns(token, config)) { new Thread(new Runnable() { @Override public void run() { diff --git a/library/src/main/java/com/qiniu/android/storage/persistent/DnsCacheFile.java b/library/src/main/java/com/qiniu/android/storage/persistent/DnsCacheFile.java new file mode 100644 index 000000000..710f4a308 --- /dev/null +++ b/library/src/main/java/com/qiniu/android/storage/persistent/DnsCacheFile.java @@ -0,0 +1,127 @@ +package com.qiniu.android.storage.persistent; + +import com.qiniu.android.storage.Recorder; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +/** + * Created by jemy on 2019/9/17. + */ + +public class DnsCacheFile implements Recorder { + + public String directory; + + public DnsCacheFile(String directory) throws IOException { + this.directory = directory; + File f = new File(directory); + if (!f.exists()) { + boolean r = f.mkdirs(); + if (!r) { + throw new IOException("mkdir failed"); + } + return; + } + if (!f.isDirectory()) { + throw new IOException("does not mkdir"); + } + } + + /** + * 设置DNS缓存 + * + * @param key 缓存文件明 + * @param data 缓存数据 + */ + @Override + public void set(String key, byte[] data) { + File file = new File(directory); + File[] fs = file.listFiles(); + if (fs.length > 0) { + for (int i = 0; i < fs.length; i++) { + del(fs[i].getName()); + } + } + + File f = new File(directory, key); + FileOutputStream fo = null; + try { + fo = new FileOutputStream(f); + fo.write(data); + } catch (IOException e) { + e.printStackTrace(); + } + if (fo != null) { + try { + fo.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + /** + * 获取缓存 + * + * @param key 缓存文件名 + */ + @Override + public byte[] get(String key) { + File f = new File(directory, key); + FileInputStream fi = null; + byte[] data = null; + int read = 0; + try { + data = new byte[(int) f.length()]; + fi = new FileInputStream(f); + read = fi.read(data); + } catch (IOException e) { + e.printStackTrace(); + } + if (fi != null) { + try { + fi.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + if (read == 0) { + return null; + } + return data; + } + + //f.delete()=false时才会有fs.length>1的情况 + public String getFileName() { + File file = new File(directory); + File[] fs = file.listFiles(); + if (fs.length == 1) { + return fs[0].getName(); + } else if (fs.length > 1) { + String fileName = null; + long cachetime = 0; + for (int i = 1; i < fs.length; i++) { + String key = fs[i].getName(); + long time = Long.parseLong(key.split(":")[0]); + if (time > cachetime) { + del(fileName); + cachetime = time; + fileName = key; + } + } + return fileName; + } + return null; + } + + @Override + public void del(String key) { + if (key != null) { + File f = new File(directory, key); + f.delete(); + } + } +} diff --git a/library/src/main/java/com/qiniu/android/storage/persistent/FileRecorder.java b/library/src/main/java/com/qiniu/android/storage/persistent/FileRecorder.java index 8cc99dfde..6d91e8ae3 100644 --- a/library/src/main/java/com/qiniu/android/storage/persistent/FileRecorder.java +++ b/library/src/main/java/com/qiniu/android/storage/persistent/FileRecorder.java @@ -121,4 +121,9 @@ public void del(String key) { File f = new File(directory, hash(key)); f.delete(); } -} + + @Override + public String getFileName() { + return null; + } +} \ No newline at end of file From 8667835a7800414030cddf79007db975ae40f2cc Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Tue, 17 Sep 2019 16:39:10 +0800 Subject: [PATCH 21/31] custom dns --- .../src/main/java/com/qiniu/android/http/Client.java | 10 +--------- .../java/com/qiniu/android/http/DnsPrefetcher.java | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/http/Client.java b/library/src/main/java/com/qiniu/android/http/Client.java index b9ce4a818..b56eab396 100644 --- a/library/src/main/java/com/qiniu/android/http/Client.java +++ b/library/src/main/java/com/qiniu/android/http/Client.java @@ -27,8 +27,6 @@ import static com.qiniu.android.http.ResponseInfo.NetworkError; -//import okhttp3.MultipartBody; - /** * Created by bailong on 15/11/12. */ @@ -57,13 +55,7 @@ public Client(ProxyConfiguration proxy, int connectTimeout, int responseTimeout, builder.dns(new okhttp3.Dns() { @Override public List lookup(String hostname) throws UnknownHostException { - if (dns != null) { - try { - return dns.lookup(hostname); - } catch (Exception e) { - e.printStackTrace(); - } - } else if (DnsPrefetcher.getDnsPrefetcher().getInetAddressByHost(hostname) != null) { + if (DnsPrefetcher.getDnsPrefetcher().getInetAddressByHost(hostname) != null) { return DnsPrefetcher.getDnsPrefetcher().getInetAddressByHost(hostname); } return okhttp3.Dns.SYSTEM.lookup(hostname); diff --git a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java index 20fc4f989..fe3f2768b 100644 --- a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java +++ b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java @@ -304,7 +304,7 @@ public static void startPrefetchDns(String token, Configuration config) { String currentTime = String.valueOf(System.currentTimeMillis()); String localip = AndroidNetwork.getHostIP(); String akAndScope = StringUtils.getAkAndScope(token); - String cacheKey = format(Locale.ENGLISH, "\"time:\":%s\"ip:\":%s\"ak\":%s", currentTime, localip, akAndScope); + String cacheKey = format(Locale.ENGLISH, "%s:%s:%s", currentTime, localip, akAndScope); Recorder recorder = null; DnsPrefetcher dnsPrefetcher = null; try { From 08c8605408d897bcd2092df1939bdd77f216bd2f Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Mon, 23 Sep 2019 19:01:01 +0800 Subject: [PATCH 22/31] recode cacheKey --- .../java/com/qiniu/android/DnsApiTest.java | 23 +++++++ .../com/qiniu/android/common/Constants.java | 2 +- .../com/qiniu/android/http/DnsPrefetcher.java | 15 +++-- .../android/http/custom/DnsCacheKey.java | 65 +++++++++++++++++++ 4 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 library/src/main/java/com/qiniu/android/http/custom/DnsCacheKey.java diff --git a/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java b/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java index befa30722..1906ee9cc 100644 --- a/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java +++ b/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java @@ -8,6 +8,7 @@ import com.qiniu.android.common.ZoneInfo; import com.qiniu.android.http.DnsPrefetcher; import com.qiniu.android.http.ResponseInfo; +import com.qiniu.android.http.custom.DnsCacheKey; import com.qiniu.android.storage.Configuration; import com.qiniu.android.storage.Recorder; import com.qiniu.android.storage.UpCompletionHandler; @@ -16,6 +17,7 @@ import com.qiniu.android.storage.UploadOptions; import com.qiniu.android.storage.persistent.DnsCacheFile; import com.qiniu.android.utils.AndroidNetwork; +import com.qiniu.android.utils.StringUtils; import org.json.JSONObject; @@ -125,7 +127,16 @@ public void testRecoverCache() { } catch (IOException e) { e.printStackTrace(); } + String fileName = recorder.getFileName(); + if (fileName == null) { + Log.e("qiniutest: ", "recover file is null "); + return; + } byte[] data = recorder.get(recorder.getFileName()); + if (data == null) { + Log.e("qiniutest: ", "recover data is null "); + return; + } DnsPrefetcher.recoverDnsCache(data); @@ -189,4 +200,16 @@ public void complete(String k, ResponseInfo rinfo, JSONObject response) { } } + public void testSerializable() { + DnsCacheKey key = new DnsCacheKey("12321", "127.0.0.1", "akscope"); + Log.e("qiniutest", key.toString()); + DnsCacheKey key1 = DnsCacheKey.toCacheKey(key.toString()); + if (key1 == null) { + return; + } + Log.e("qiniutest", key1.getCurrentTime() + ":" + key1.getLocalIp() + ":" + key1.getAkScope()); + + } + + } diff --git a/library/src/main/java/com/qiniu/android/common/Constants.java b/library/src/main/java/com/qiniu/android/common/Constants.java index 3a02de06f..35ff24b6f 100644 --- a/library/src/main/java/com/qiniu/android/common/Constants.java +++ b/library/src/main/java/com/qiniu/android/common/Constants.java @@ -2,7 +2,7 @@ public final class Constants { - public static final String VERSION = "7.3.18"; + public static final String VERSION = "7.4.0"; public static final String UTF_8 = "utf-8"; } diff --git a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java index fe3f2768b..b1fbe9218 100644 --- a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java +++ b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java @@ -4,6 +4,7 @@ import com.qiniu.android.common.Constants; import com.qiniu.android.common.FixedZone; import com.qiniu.android.common.ZoneInfo; +import com.qiniu.android.http.custom.DnsCacheKey; import com.qiniu.android.storage.Configuration; import com.qiniu.android.storage.Recorder; import com.qiniu.android.storage.persistent.DnsCacheFile; @@ -279,16 +280,16 @@ public static boolean checkRePrefetchDns(String token, Configuration config) { if (data == null) return true; - String[] cacheKey = dnscache.split(":"); - if (cacheKey.length < 3) + DnsCacheKey cacheKey = DnsCacheKey.toCacheKey(dnscache); + if (cacheKey == null) return true; String currentTime = String.valueOf(System.currentTimeMillis()); String localip = AndroidNetwork.getHostIP(); - String akAndScope = StringUtils.getAkAndScope(token); + String akScope = StringUtils.getAkAndScope(token); - long cacheTime = (Long.parseLong(currentTime) - Long.parseLong(cacheKey[0])) / 1000; - if (!localip.equals(cacheKey[1]) || cacheTime > config.dnsCacheTimeMs || !akAndScope.equals(cacheKey[2])) { + long cacheTime = (Long.parseLong(currentTime) - Long.parseLong(cacheKey.getCurrentTime())) / 1000; + if (!localip.equals(cacheKey.getLocalIp()) || cacheTime > config.dnsCacheTimeMs || !akScope.equals(cacheKey.getAkScope())) { return true; } @@ -303,8 +304,8 @@ public static boolean checkRePrefetchDns(String token, Configuration config) { public static void startPrefetchDns(String token, Configuration config) { String currentTime = String.valueOf(System.currentTimeMillis()); String localip = AndroidNetwork.getHostIP(); - String akAndScope = StringUtils.getAkAndScope(token); - String cacheKey = format(Locale.ENGLISH, "%s:%s:%s", currentTime, localip, akAndScope); + String akScope = StringUtils.getAkAndScope(token); + String cacheKey = new DnsCacheKey(currentTime, localip, akScope).toString(); Recorder recorder = null; DnsPrefetcher dnsPrefetcher = null; try { diff --git a/library/src/main/java/com/qiniu/android/http/custom/DnsCacheKey.java b/library/src/main/java/com/qiniu/android/http/custom/DnsCacheKey.java new file mode 100644 index 000000000..b148a462a --- /dev/null +++ b/library/src/main/java/com/qiniu/android/http/custom/DnsCacheKey.java @@ -0,0 +1,65 @@ +package com.qiniu.android.http.custom; + + +import org.json.JSONException; +import org.json.JSONObject; + +/** + * Created by jemy on 2019/9/23. + */ + +public class DnsCacheKey { + public String currentTime; + public String localIp; + public String akScope; + + public DnsCacheKey() { + + } + + public DnsCacheKey(String currentTime, String localIp, String akScope) { + this.currentTime = currentTime; + this.localIp = localIp; + this.akScope = akScope; + } + + public String getCurrentTime() { + return currentTime; + } + + public String getAkScope() { + return akScope; + } + + public String getLocalIp() { + return localIp; + } + + public void setAkScope(String akScope) { + this.akScope = akScope; + } + + public void setCurrentTime(String currentTime) { + this.currentTime = currentTime; + } + + public void setLocalIp(String localIp) { + this.localIp = localIp; + } + + public static DnsCacheKey toCacheKey(String key) { + try { + JSONObject object = new JSONObject(key); + return new DnsCacheKey(object.getString("currentTime"), object.getString("localIp"), object.getString("akScope")); + } catch (JSONException e) { + e.printStackTrace(); + return null; + } + } + + @Override + public String toString() { + return "{\"currentTime\":\"" + currentTime + "\", \"localIp\":\"" + localIp + "\", \"akScope\":\"" + akScope + "\"}"; + + } +} From 2f03fb67a219bebcb8587b15b208e34325b31194 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Mon, 23 Sep 2019 19:05:56 +0800 Subject: [PATCH 23/31] format code test --- .../src/androidTest/java/com/qiniu/android/DnsApiTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java b/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java index 1906ee9cc..7404a3453 100644 --- a/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java +++ b/library/src/androidTest/java/com/qiniu/android/DnsApiTest.java @@ -95,10 +95,10 @@ public void testLocalIp() { public void testDnsPreAndcache() { Configuration config = new Configuration.Builder().build(); - boolean needPrefetch = DnsPrefetcher.checkRePrefetchDns("MP_Ebql_lSsUrDr7WrXn_5vKocQDLvTPCNEFeVmp:5mVFMc75Yy4nWYJ8E5j5ESW51Rs=:eyJzY29wZSI6ImFuZHJvaWR0ZXN0IiwiZGVhZGxpbmUiOjE1Njg3MDcxOTl9", config); + boolean needPrefetch = DnsPrefetcher.checkRePrefetchDns(TestConfig.uptoken_prefetch, config); Log.e("qiniutest", "check:" + needPrefetch); if (needPrefetch) { - DnsPrefetcher.startPrefetchDns("MP_Ebql_lSsUrDr7WrXn_5vKocQDLvTPCNEFeVmp:5mVFMc75Yy4nWYJ8E5j5ESW51Rs=:eyJzY29wZSI6ImFuZHJvaWR0ZXN0IiwiZGVhZGxpbmUiOjE1Njg3MDcxOTl9", config); + DnsPrefetcher.startPrefetchDns(TestConfig.uptoken_prefetch, config); } else { testRecoverCache(); return; From 3971a0346ad39c1fddab34a61e34b15c29cbc8c5 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Wed, 9 Oct 2019 16:17:45 +0800 Subject: [PATCH 24/31] safety --- .../android/storage/ResumeUploaderFast.java | 23 ++++++++++--------- .../qiniu/android/utils/AndroidNetwork.java | 5 ++-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java index 216fc2901..bea7cf1f7 100644 --- a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java +++ b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java @@ -24,6 +24,7 @@ import java.util.LinkedHashMap; import java.util.Locale; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import static java.lang.String.format; @@ -90,7 +91,7 @@ public class ResumeUploaderFast implements Runnable { /** * 重传域名数 */ - volatile int retried = 0; + AtomicInteger retried = new AtomicInteger(0); /** * 单域名检测次数 */ @@ -347,9 +348,9 @@ public void complete(ResponseInfo info, JSONObject response) { } // mkfile ，允许多重试一次，这里不需要重试时，成功与否都complete回调给客户端 - if (info.needRetry() && retried < config.retryMax + 1) { + if (info.needRetry() && retried.get() < config.retryMax + 1) { makeFile(upHost, getMkfileCompletionHandler(), options.cancellationSignal); - retried += 1; + retried.addAndGet(1); return; } completionHandler.complete(key, info, response); @@ -442,10 +443,10 @@ && checkRetried())) { offsets[(int) (offset / Configuration.BLOCK_SIZE)] = offset; record(offsets); upBlock += 1; - } - if (upBlock == tblock) { - makeFile(upHost, getMkfileCompletionHandler(), options.cancellationSignal); - return; + if (upBlock == tblock) { + makeFile(upHost, getMkfileCompletionHandler(), options.cancellationSignal); + return; + } } if (blockInfo.size() > 0) { @@ -460,16 +461,16 @@ && checkRetried())) { private synchronized void updateRetried() { if (singleDomainRetry < config.retryMax) { singleDomainRetry += 1; - } else if (retried < domainRetry) { + } else if (retried.get() < domainRetry) { singleDomainRetry = 1; - retried += 1; + retried.getAndAdd(1); upHost = config.zone.upHost(token.token, config.useHttps, upHost); } } - private synchronized boolean checkRetried() { - return retried < domainRetry; + private boolean checkRetried() { + return retried.get() < domainRetry; } private boolean isChunkOK(ResponseInfo info, JSONObject response) { diff --git a/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java b/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java index 9959c050e..935d4c173 100644 --- a/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java +++ b/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java @@ -50,7 +50,8 @@ public static String getHostIP() { if (ia instanceof Inet6Address) { ipv6 = checkIpv6(ia.getHostAddress()); if (ipv6 != null) { - return ipv6; + hostIp = ipv6; + break; } continue; } @@ -74,7 +75,7 @@ public static String getHostIP() { * @return */ private static String checkIpv6(String ipv6) { - if (ipv6 != null && ipv6.contains("%")) { + if (ipv6 != null) { String[] split = ipv6.split("%"); String s1 = split[0]; if (s1 != null && s1.contains(":")) { From 31fefbd2993aa795ec4b298dbdcf7a8bac458ed3 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Wed, 9 Oct 2019 19:43:56 +0800 Subject: [PATCH 25/31] safety --- .../android/storage/ResumeUploaderFast.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java index bea7cf1f7..ecc3dd7da 100644 --- a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java +++ b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java @@ -87,7 +87,7 @@ public class ResumeUploaderFast implements Runnable { /** * 总块数 */ - private int tblock; + AtomicInteger tblock; /** * 重传域名数 */ @@ -95,7 +95,7 @@ public class ResumeUploaderFast implements Runnable { /** * 单域名检测次数 */ - volatile int singleDomainRetry = 1; + AtomicInteger singleDomainRetry = new AtomicInteger(0); /** * 线程数量 */ @@ -152,9 +152,9 @@ public void complete(String key, ResponseInfo info, JSONObject response) { } }; this.options = options != null ? options : UploadOptions.defaultOptions(); - tblock = (int) (totalSize + Configuration.BLOCK_SIZE - 1) / Configuration.BLOCK_SIZE; - this.offsets = new Long[tblock]; - contexts = new String[tblock]; + tblock = new AtomicInteger((int) (totalSize + Configuration.BLOCK_SIZE - 1) / Configuration.BLOCK_SIZE); + this.offsets = new Long[tblock.get()]; + contexts = new String[tblock.get()]; modifyTime = f.lastModified(); this.token = token; this.blockInfo = new LinkedHashMap<>(); @@ -187,7 +187,7 @@ public void run() { */ private void putBlockInfo() { Long[] offs = recoveryFromRecord(); - int lastBlock = tblock - 1; + int lastBlock = tblock.get() - 1; if (offs == null) { for (int i = 0; i < lastBlock; i++) { blockInfo.put((long) i * Configuration.BLOCK_SIZE, Configuration.BLOCK_SIZE); @@ -443,7 +443,7 @@ && checkRetried())) { offsets[(int) (offset / Configuration.BLOCK_SIZE)] = offset; record(offsets); upBlock += 1; - if (upBlock == tblock) { + if (upBlock == tblock.get()) { makeFile(upHost, getMkfileCompletionHandler(), options.cancellationSignal); return; } @@ -459,10 +459,10 @@ && checkRetried())) { } private synchronized void updateRetried() { - if (singleDomainRetry < config.retryMax) { - singleDomainRetry += 1; + if (singleDomainRetry.get() < config.retryMax) { + singleDomainRetry.getAndAdd(1); } else if (retried.get() < domainRetry) { - singleDomainRetry = 1; + singleDomainRetry.getAndSet(1); retried.getAndAdd(1); upHost = config.zone.upHost(token.token, config.useHttps, upHost); } From 56ebbbbe5da474c50bba026c3c306bad3a4e1831 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Thu, 10 Oct 2019 16:14:59 +0800 Subject: [PATCH 26/31] some func have been optimised --- library/library.iml | 15 ++++---- .../com/qiniu/android/common/AutoZone.java | 11 +++++- .../com/qiniu/android/http/DnsPrefetcher.java | 35 +++++++++---------- .../com/qiniu/android/http/ResponseInfo.java | 3 +- .../android/storage/ResumeUploaderFast.java | 28 +++++++-------- 5 files changed, 49 insertions(+), 43 deletions(-) diff --git a/library/library.iml b/library/library.iml index d44595374..d72d61f22 100644 --- a/library/library.iml +++ b/library/library.iml @@ -61,13 +61,6 @@ - - - - - - - @@ -75,6 +68,13 @@ + + + + + + + @@ -99,7 +99,6 @@ - diff --git a/library/src/main/java/com/qiniu/android/common/AutoZone.java b/library/src/main/java/com/qiniu/android/common/AutoZone.java index 267fb2cf2..fccc22d92 100644 --- a/library/src/main/java/com/qiniu/android/common/AutoZone.java +++ b/library/src/main/java/com/qiniu/android/common/AutoZone.java @@ -21,7 +21,7 @@ public final class AutoZone extends Zone { /** * 自动判断机房 */ - private final String ucServer; + private String ucServer; private Map zones = new ConcurrentHashMap<>(); private Client client = new Client(); @@ -40,6 +40,15 @@ public AutoZone(boolean useHttps) { } } + //私有云可能改变ucServer + public void setUcServer(String ucServer) { + this.ucServer = ucServer; + } + + public String getUcServer() { + return this.ucServer; + } + private void getZoneJsonAsync(ZoneIndex index, CompletionHandler handler) { String address = ucServer + "/v2/query?ak=" + index.accessKey + "&bucket=" + index.bucket; client.asyncGet(address, null, UpToken.NULL, handler); diff --git a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java index b1fbe9218..f8e0294e1 100644 --- a/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java +++ b/library/src/main/java/com/qiniu/android/http/DnsPrefetcher.java @@ -22,11 +22,8 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Locale; import java.util.concurrent.ConcurrentHashMap; -import static java.lang.String.format; - /** *

* Created by jemy on 2019/8/20. @@ -95,17 +92,17 @@ private void preHosts() { //preQuery sync ZoneInfo zoneInfo = getPreQueryZone(); if (zoneInfo != null) { - for (int i = 0; i < zoneInfo.upDomainsList.size(); i++) { - if (set.add(zoneInfo.upDomainsList.get(i))) - mHosts.add(zoneInfo.upDomainsList.get(i)); + for (String host : zoneInfo.upDomainsList) { + if (set.add(host)) + mHosts.add(host); } } //local List listZoneinfo = getLocalZone(); for (ZoneInfo zone : listZoneinfo) { - for (int i = 0; i < zone.upDomainsList.size(); i++) { - if (set.add(zone.upDomainsList.get(i))) - mHosts.add(zone.upDomainsList.get(i)); + for (String host : zone.upDomainsList) { + if (set.add(host)) + mHosts.add(host); } } if (set.add(Config.preQueryHost)) @@ -115,14 +112,14 @@ private void preHosts() { private void preFetch() { List rePreHosts = new ArrayList(); - for (int i = 0; i < mHosts.size(); i++) { + for (String host : mHosts) { List inetAddresses = null; try { - inetAddresses = okhttp3.Dns.SYSTEM.lookup(mHosts.get(i)); - mConcurrentHashMap.put(mHosts.get(i), inetAddresses); + inetAddresses = okhttp3.Dns.SYSTEM.lookup(host); + mConcurrentHashMap.put(host, inetAddresses); } catch (UnknownHostException e) { e.printStackTrace(); - rePreHosts.add(mHosts.get(i)); + rePreHosts.add(host); } } rePreFetch(rePreHosts, null); @@ -135,11 +132,11 @@ private void preFetch() { * @param customeDns 是否自定义dns */ private void rePreFetch(List rePreHosts, Dns customeDns) { - for (int i = 0; i < rePreHosts.size(); i++) { + for (String host : rePreHosts) { int rePreNum = 0; while (rePreNum < Config.rePreHost) { rePreNum += 1; - if (rePreFetch(rePreHosts.get(i), customeDns)) + if (rePreFetch(host, customeDns)) break; } } @@ -170,14 +167,14 @@ private boolean rePreFetch(String host, Dns customeDns) { */ public void dnsPreByCustom(Dns dns) { List rePreHosts = new ArrayList(); - for (int i = 0; i < mHosts.size(); i++) { + for (String host : mHosts) { List inetAddresses = null; try { - inetAddresses = dns.lookup(mHosts.get(i)); - mConcurrentHashMap.put(mHosts.get(i), inetAddresses); + inetAddresses = dns.lookup(host); + mConcurrentHashMap.put(host, inetAddresses); } catch (UnknownHostException e) { e.printStackTrace(); - rePreHosts.add(mHosts.get(i)); + rePreHosts.add(host); } } rePreFetch(rePreHosts, dns); diff --git a/library/src/main/java/com/qiniu/android/http/ResponseInfo.java b/library/src/main/java/com/qiniu/android/http/ResponseInfo.java index 152f5ce37..aad0e258d 100644 --- a/library/src/main/java/com/qiniu/android/http/ResponseInfo.java +++ b/library/src/main/java/com/qiniu/android/http/ResponseInfo.java @@ -200,7 +200,8 @@ public static ResponseInfo fileError(Exception e, final UpToken upToken) { } public static ResponseInfo networkError(int code, UpToken upToken) { - return create(null, code, "", "", "", "", "", "", 80, 0, 0, "Network error during preQuery, Please use http try again", upToken, 0); + return create(null, code, "", "", "", "", "", "", 80, 0, 0, "Network error during preQuery, Please check your network or " + + "use http try again", upToken, 0); } public static boolean isStatusCodeForBrokenNetwork(int code) { diff --git a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java index ecc3dd7da..40108f04c 100644 --- a/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java +++ b/library/src/main/java/com/qiniu/android/storage/ResumeUploaderFast.java @@ -25,6 +25,7 @@ import java.util.Locale; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import static java.lang.String.format; @@ -83,7 +84,7 @@ public class ResumeUploaderFast implements Runnable { /** * 上传域名 */ - volatile String upHost; + AtomicReference upHost = new AtomicReference(); /** * 总块数 */ @@ -171,13 +172,13 @@ public void run() { } putBlockInfo(); - upHost = config.zone.upHost(token.token, config.useHttps, null); + upHost.set(config.zone.upHost(token.token, config.useHttps, null)); if (blockInfo.size() < multithread) { multithread = blockInfo.size(); } for (int i = 0; i < multithread; i++) { BlockElement mblock = getBlockInfo(); - new UploadThread(mblock.getOffset(), mblock.getBlocksize(), upHost).start(); + new UploadThread(mblock.getOffset(), mblock.getBlocksize(), upHost.get().toString()).start(); } } @@ -313,8 +314,8 @@ private ProgressHandler getProgressHandler() { @Override public void onProgress(long bytesWritten, long totalSize) { long size = 0; - for (int i = 0; i < offsets.length; i++) { - if (offsets[i] != null && offsets[i] > 0) { + for (Long offset : offsets) { + if (offset != null && offset > 0) { size += 1; } } @@ -349,7 +350,7 @@ public void complete(ResponseInfo info, JSONObject response) { // mkfile ，允许多重试一次，这里不需要重试时，成功与否都complete回调给客户端 if (info.needRetry() && retried.get() < config.retryMax + 1) { - makeFile(upHost, getMkfileCompletionHandler(), options.cancellationSignal); + makeFile(upHost.get().toString(), getMkfileCompletionHandler(), options.cancellationSignal); retried.addAndGet(1); return; } @@ -384,14 +385,14 @@ public void complete(ResponseInfo info, JSONObject response) { if (!isChunkOK(info, response)) { if (info.statusCode == 701 && checkRetried()) { updateRetried(); - mkblk(offset, blockSize, upHost); + mkblk(offset, blockSize, upHost.get().toString()); return; } if (upHost != null && ((isNotChunkToQiniu(info, response) || info.needRetry()) && checkRetried())) { updateRetried(); - mkblk(offset, blockSize, upHost); + mkblk(offset, blockSize, upHost.get().toString()); return; } @@ -403,8 +404,7 @@ && checkRetried())) { String context = null; if (response == null && checkRetried()) { updateRetried(); - mkblk(offset, blockSize, upHost); - + mkblk(offset, blockSize, upHost.get().toString()); return; } long crc = 0; @@ -418,7 +418,7 @@ && checkRetried())) { } if ((context == null || crc != crc32) && checkRetried()) { updateRetried(); - mkblk(offset, blockSize, upHost); + mkblk(offset, blockSize, upHost.get().toString()); return; } if (context == null) { @@ -444,7 +444,7 @@ && checkRetried())) { record(offsets); upBlock += 1; if (upBlock == tblock.get()) { - makeFile(upHost, getMkfileCompletionHandler(), options.cancellationSignal); + makeFile(upHost.get().toString(), getMkfileCompletionHandler(), options.cancellationSignal); return; } } @@ -452,7 +452,7 @@ && checkRetried())) { if (blockInfo.size() > 0) { BlockElement mblock = getBlockInfo(); if (mblock.getOffset() != 0 && mblock.getBlocksize() != 0) - new UploadThread(mblock.getOffset(), mblock.getBlocksize(), upHost).start(); + new UploadThread(mblock.getOffset(), mblock.getBlocksize(), upHost.get().toString()).start(); } } }; @@ -464,7 +464,7 @@ private synchronized void updateRetried() { } else if (retried.get() < domainRetry) { singleDomainRetry.getAndSet(1); retried.getAndAdd(1); - upHost = config.zone.upHost(token.token, config.useHttps, upHost); + upHost.getAndSet(config.zone.upHost(token.token, config.useHttps, upHost.get().toString())); } } From a6c4ee5ee340dce2affda199d98c38e151627c0d Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Thu, 10 Oct 2019 18:04:41 +0800 Subject: [PATCH 27/31] rename atomic --- .../java/com/qiniu/android/storage/UploadManager.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/storage/UploadManager.java b/library/src/main/java/com/qiniu/android/storage/UploadManager.java index ec520675a..78b6f0da9 100644 --- a/library/src/main/java/com/qiniu/android/storage/UploadManager.java +++ b/library/src/main/java/com/qiniu/android/storage/UploadManager.java @@ -25,7 +25,10 @@ public final class UploadManager { private final Client client; private int multithreads = 1; private static int DEF_THREAD_NUM = 3; - static AtomicBoolean atomic = new AtomicBoolean(false); + /** + * 保证代码只执行一次，防止多个uploadManager同时开始预取dns + */ + static AtomicBoolean atomicStruct = new AtomicBoolean(false); /** * default 3 Threads @@ -139,7 +142,7 @@ public void put(final byte[] data, final String key, final String token, return; } - if (atomic.compareAndSet(false, true)) { + if (atomicStruct.compareAndSet(false, true)) { if (DnsPrefetcher.checkRePrefetchDns(token, config)) { new Thread(new Runnable() { @Override @@ -198,7 +201,7 @@ public void put(final File file, final String key, final String token, final UpC return; } - if (atomic.compareAndSet(false, true)) { + if (atomicStruct.compareAndSet(false, true)) { if (DnsPrefetcher.checkRePrefetchDns(token, config)) { new Thread(new Runnable() { @Override From 5d4ff72d617cf8c69c11685166f14cdf2273ccaf Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Fri, 11 Oct 2019 15:14:03 +0800 Subject: [PATCH 28/31] checkIpv6 --- .../main/java/com/qiniu/android/utils/AndroidNetwork.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java b/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java index 935d4c173..57dd20765 100644 --- a/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java +++ b/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java @@ -48,9 +48,9 @@ public static String getHostIP() { while (ias.hasMoreElements()) { ia = ias.nextElement(); if (ia instanceof Inet6Address) { - ipv6 = checkIpv6(ia.getHostAddress()); - if (ipv6 != null) { - hostIp = ipv6; + //ipv6 = checkIpv6(ia.getHostAddress()); + if (!ia.isLinkLocalAddress()&&!ia.isLoopbackAddress()) { + hostIp = ia.getHostAddress(); break; } continue; From 786dab37d72efea457574ba9dec9acdf39aa3eb5 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Fri, 11 Oct 2019 16:20:13 +0800 Subject: [PATCH 29/31] recode get ip --- .../qiniu/android/utils/AndroidNetwork.java | 40 ++----------------- 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java b/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java index 57dd20765..665493757 100644 --- a/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java +++ b/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java @@ -38,28 +38,19 @@ public static boolean isNetWorkReady() { */ public static String getHostIP() { String hostIp = null; - String ipv6 = null; try { Enumeration nis = NetworkInterface.getNetworkInterfaces(); InetAddress ia = null; while (nis.hasMoreElements()) { NetworkInterface ni = (NetworkInterface) nis.nextElement(); Enumeration ias = ni.getInetAddresses(); - while (ias.hasMoreElements()) { - ia = ias.nextElement(); - if (ia instanceof Inet6Address) { - //ipv6 = checkIpv6(ia.getHostAddress()); - if (!ia.isLinkLocalAddress()&&!ia.isLoopbackAddress()) { - hostIp = ia.getHostAddress(); - break; - } - continue; - } - String ip = ia.getHostAddress(); - if (!"127.0.0.1".equals(ip)) { + ia = ias.nextElement(); + if (ia instanceof Inet6Address) { + if (!ia.isLinkLocalAddress() && !ia.isLoopbackAddress()) { hostIp = ia.getHostAddress(); break; } + continue; } } } catch (SocketException e) { @@ -68,27 +59,4 @@ public static String getHostIP() { return hostIp; } - /** - * first segment contains "fe" or "fc": https://blog.csdn.net/fdl19881/article/details/7091138 - * - * @param ipv6 - * @return - */ - private static String checkIpv6(String ipv6) { - if (ipv6 != null) { - String[] split = ipv6.split("%"); - String s1 = split[0]; - if (s1 != null && s1.contains(":")) { - String[] split1 = s1.split(":"); - if (split1.length == 6 || split1.length == 8) { - if (split1[0].contains("fe") || split1[0].contains("fc")) { - return null; - } else { - return s1; - } - } - } - } - return null; - } } From 376f3f7a14de220a807732adaf95b625dac44959 Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Sat, 12 Oct 2019 10:14:33 +0800 Subject: [PATCH 30/31] getHostIp --- .../src/main/java/com/qiniu/android/utils/AndroidNetwork.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java b/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java index 665493757..511669877 100644 --- a/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java +++ b/library/src/main/java/com/qiniu/android/utils/AndroidNetwork.java @@ -44,8 +44,8 @@ public static String getHostIP() { while (nis.hasMoreElements()) { NetworkInterface ni = (NetworkInterface) nis.nextElement(); Enumeration ias = ni.getInetAddresses(); - ia = ias.nextElement(); - if (ia instanceof Inet6Address) { + while (ias.hasMoreElements()) { + ia = ias.nextElement(); if (!ia.isLinkLocalAddress() && !ia.isLoopbackAddress()) { hostIp = ia.getHostAddress(); break; From a262f76e16084914e71f7dd0fbd6ad3fdb828c7a Mon Sep 17 00:00:00 2001 From: JemyCheung Date: Sat, 12 Oct 2019 13:05:59 +0800 Subject: [PATCH 31/31] test --- .../androidTest/java/com/qiniu/android/CancelTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/library/src/androidTest/java/com/qiniu/android/CancelTest.java b/library/src/androidTest/java/com/qiniu/android/CancelTest.java index 5c732ef11..056f01c32 100644 --- a/library/src/androidTest/java/com/qiniu/android/CancelTest.java +++ b/library/src/androidTest/java/com/qiniu/android/CancelTest.java @@ -44,10 +44,10 @@ public void testFile() throws Throwable { checkTemp(ts, "testFile"); } - public void testData() throws Throwable { - Temp[] ts = new Temp[]{templateData(400, 0.2), templateData(700, 0.2), templateData(1024, 0.51), templateData(4 * 1024 + 785, 0.5), templateData(4 * 1024, 0.5), templateData(8 * 1024, 0.6)}; - checkTemp(ts, "testData"); - } +// public void testData() throws Throwable { +// Temp[] ts = new Temp[]{templateData(400, 0.2), templateData(700, 0.2), templateData(1024, 0.51), templateData(4 * 1024 + 785, 0.5), templateData(4 * 1024, 0.5), templateData(8 * 1024, 0.6)}; +// checkTemp(ts, "testData"); +// } private void checkTemp(Temp[] ts, String type) { int failedCount = 0;