From 06d72b769962cb30f26a740495b3f5299f36be46 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 13 Jul 2021 16:38:25 +0800 Subject: [PATCH 01/20] local fetch not contain uc --- .../qiniu/android/http/dns/DnsPrefetcher.java | 46 ++++++++----------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/http/dns/DnsPrefetcher.java b/library/src/main/java/com/qiniu/android/http/dns/DnsPrefetcher.java index 34181b70..b34d0b14 100644 --- a/library/src/main/java/com/qiniu/android/http/dns/DnsPrefetcher.java +++ b/library/src/main/java/com/qiniu/android/http/dns/DnsPrefetcher.java @@ -71,37 +71,35 @@ public boolean recoverCache(){ } public void localFetch(){ - if (!prepareToPreFetch()){ - return; - } - - String[] hosts = getLocalPreHost(); - synchronized (this) { - prefetchHosts.addAll(Arrays.asList(hosts)); - } - preFetchHosts(hosts); - recorderDnsCache(); - endPreFetch(); + addPreFetchHosts(getLocalPreHost()); } public boolean checkAndPrefetchDnsIfNeed(Zone currentZone, UpToken token){ - if (!prepareToPreFetch()){ - return false; - } + return addPreFetchHosts(getCurrentZoneHosts(currentZone, token)); + } - String[] hosts = getCurrentZoneHosts(currentZone, token); + public boolean addPreFetchHosts(String[] hosts) { if (hosts == null) { return false; } + // 已经添加则不再触发预取 + boolean prefetchHostsContainHosts = false; synchronized (this) { + int countBeforeAdd = prefetchHosts.size(); prefetchHosts.addAll(Arrays.asList(hosts)); + int countAfterAdd = prefetchHosts.size(); + if (countAfterAdd > countBeforeAdd) { + prefetchHostsContainHosts = true; + } } - preFetchHosts(hosts); - recorderDnsCache(); - endPreFetch(); - return true; + if (prefetchHostsContainHosts) { + checkWhetherCachedDnsValid(); + return true; + } else { + return false; + } } public void invalidNetworkAddress(IDnsNetworkAddress address){ @@ -285,15 +283,7 @@ private void clearPreHosts(){ private String[] getLocalPreHost(){ - ArrayList localHosts = new ArrayList<>(); - - localHosts.add(Config.preQueryHost00); - localHosts.add(Config.preQueryHost01); - - String logReport = Config.upLogURL; - localHosts.add(logReport); - - return localHosts.toArray(new String[0]); + return new String[]{Config.upLogURL}; } private String[] getCurrentZoneHosts(Zone currentZone, UpToken token){ From 860d0d96133234056a380d63dc6878ced2a35929 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 13 Jul 2021 17:06:36 +0800 Subject: [PATCH 02/20] optimize dns prefetch of uc --- .../com/qiniu/android/common/AutoZone.java | 10 ++++++++++ .../http/dns/DnsPrefetchTransaction.java | 20 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/library/src/main/java/com/qiniu/android/common/AutoZone.java b/library/src/main/java/com/qiniu/android/common/AutoZone.java index d056fd89..1902eafc 100644 --- a/library/src/main/java/com/qiniu/android/common/AutoZone.java +++ b/library/src/main/java/com/qiniu/android/common/AutoZone.java @@ -1,6 +1,7 @@ package com.qiniu.android.common; import com.qiniu.android.http.ResponseInfo; +import com.qiniu.android.http.dns.DnsPrefetchTransaction; import com.qiniu.android.http.request.RequestTransaction; import com.qiniu.android.http.metrics.UploadRegionRequestMetrics; import com.qiniu.android.storage.UpToken; @@ -44,6 +45,14 @@ public List getUcServerList() { } } + private String[] getUcServerArray() { + if (ucServer != null) { + return new String[]{ucServer}; + } else { + return new String[]{Config.preQueryHost00, Config.preQueryHost01}; + } + } + @Override public ZonesInfo getZonesInfo(UpToken token) { if (token == null) { @@ -75,6 +84,7 @@ public void preQuery(final UpToken token, final QueryHandler completeHandler) { return; } + DnsPrefetchTransaction.addDnsCheckAndPrefetchTransaction(getUcServerArray()); try { SingleFlight.perform(cacheKey, new SingleFlight.ActionHandler() { diff --git a/library/src/main/java/com/qiniu/android/http/dns/DnsPrefetchTransaction.java b/library/src/main/java/com/qiniu/android/http/dns/DnsPrefetchTransaction.java index 067aa9aa..d2c9b664 100644 --- a/library/src/main/java/com/qiniu/android/http/dns/DnsPrefetchTransaction.java +++ b/library/src/main/java/com/qiniu/android/http/dns/DnsPrefetchTransaction.java @@ -58,6 +58,26 @@ public void run() { return true; } + public static synchronized boolean addDnsCheckAndPrefetchTransaction(final String[] hosts) { + if (!DnsPrefetcher.getInstance().isDnsOpen()) { + return false; + } + + if (hosts == null || hosts.length == 0) { + return false; + } + + TransactionManager manager = TransactionManager.getInstance(); + TransactionManager.Transaction loadDns = new TransactionManager.Transaction(null, 0, new Runnable() { + @Override + public void run() { + DnsPrefetcher.getInstance().addPreFetchHosts(hosts); + } + }); + manager.addTransaction(loadDns); + return true; + } + public static synchronized boolean setDnsCheckWhetherCachedValidTransactionAction() { if (!DnsPrefetcher.getInstance().isDnsOpen()) { From ba98bf7bf03f43357f2905280cd0d5c2bbb6e0dc Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Tue, 13 Jul 2021 18:58:45 +0800 Subject: [PATCH 03/20] custom dns server --- .../http/dns/DnsCustomServersResolver.java | 91 +++++++++++++++++++ .../com/qiniu/android/http/dns/HappyDns.java | 16 +++- .../com/qiniu/android/http/dns/SystemDns.java | 40 +++++++- .../request/httpclient/SystemHttpClient.java | 1 + .../android/storage/GlobalConfiguration.java | 10 ++ 5 files changed, 149 insertions(+), 9 deletions(-) create mode 100644 library/src/main/java/com/qiniu/android/http/dns/DnsCustomServersResolver.java diff --git a/library/src/main/java/com/qiniu/android/http/dns/DnsCustomServersResolver.java b/library/src/main/java/com/qiniu/android/http/dns/DnsCustomServersResolver.java new file mode 100644 index 00000000..1a398f53 --- /dev/null +++ b/library/src/main/java/com/qiniu/android/http/dns/DnsCustomServersResolver.java @@ -0,0 +1,91 @@ +package com.qiniu.android.http.dns; + +import com.qiniu.android.dns.Domain; +import com.qiniu.android.dns.IResolver; +import com.qiniu.android.dns.NetworkInfo; +import com.qiniu.android.dns.Record; +import com.qiniu.android.dns.local.Resolver; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +public class DnsCustomServersResolver implements IResolver { + + public ArrayList customServers = new ArrayList<>(); + + public DnsCustomServersResolver(String[] dnsServers, int timeout) { + if (timeout < 0 || timeout > 20) { + timeout = 2; + } + + for (String dnsServer : dnsServers) { + try { + InetAddress address = InetAddress.getByName(dnsServer); + customServers.add(new CustomServer(address, timeout)); + } catch (Exception ignore) { + } + + } + } + + @Override + public Record[] resolve(Domain domain, NetworkInfo info) throws IOException { + List servers = getCustomServers(); + + Record[] records = null; + for (CustomServer server : servers) { + records = server.resolver.resolve(domain, info); + if (records != null && records.length > 0) { + server.addReliability(); + break; + } + } + + return records; + } + + + public List getCustomServers() { + List servers = (List) customServers.clone(); + Comparator comparator = new Comparator() { + @Override + public int compare(CustomServer o1, CustomServer o2) { + if (o1.isMoreReliabilityThan(o2)) { + return -1; + } else { + return 1; + } + } + }; + + Collections.sort(servers, comparator); + return servers; + } + + public static class CustomServer { + + private int reliability = 0; + private IResolver resolver; + + CustomServer(InetAddress address, int timeout) { + this.resolver = new Resolver(address, timeout); + } + + public void addReliability() { + reliability += 1; + } + + boolean isMoreReliabilityThan(CustomServer customServer) { + return reliability > customServer.reliability; + } + + @Override + public String toString() { + return "{" + "\"" + "reliability" + "\":" + reliability + "}"; + } + } +} diff --git a/library/src/main/java/com/qiniu/android/http/dns/HappyDns.java b/library/src/main/java/com/qiniu/android/http/dns/HappyDns.java index ca936d6f..a014dbb9 100644 --- a/library/src/main/java/com/qiniu/android/http/dns/HappyDns.java +++ b/library/src/main/java/com/qiniu/android/http/dns/HappyDns.java @@ -6,6 +6,7 @@ import com.qiniu.android.dns.NetworkInfo; import com.qiniu.android.dns.Record; import com.qiniu.android.dns.http.DnspodFree; +import com.qiniu.android.storage.GlobalConfiguration; import com.qiniu.android.utils.Utils; import java.io.IOException; @@ -23,9 +24,11 @@ public class HappyDns implements Dns { public HappyDns(){ - IResolver[] resolvers = new IResolver[2]; - resolvers[0] = new SystemResolver(); - resolvers[1] = new DnspodFree(); + int dnsTimeout = GlobalConfiguration.getInstance().dnsServerResolveTimeout; + IResolver[] resolvers = new IResolver[3]; + resolvers[0] = new SystemResolver(dnsTimeout); + resolvers[1] = new DnsCustomServersResolver(GlobalConfiguration.getInstance().dnsServers, dnsTimeout); + resolvers[2] = new DnspodFree("119.29.29.29", dnsTimeout); dnsManager = new DnsManager(NetworkInfo.normal, resolvers); } @@ -66,13 +69,18 @@ public List lookup(String hostname) throws UnknownHostExcept private static class SystemResolver implements IResolver{ + private int timeout; + SystemResolver(int timeout) { + this.timeout = timeout; + } + @Override public Record[] resolve(Domain domain, NetworkInfo info) throws IOException { long timestamp = Utils.currentTimestamp(); int ttl = 120; ArrayList records = new ArrayList<>(); - List inetAddresses = new SystemDns().lookupInetAddress(domain.domain); + List inetAddresses = new SystemDns(timeout).lookupInetAddress(domain.domain); for(InetAddress inetAddress : inetAddresses){ Record record = new Record(inetAddress.getHostAddress(), Record.TYPE_A, ttl, timestamp, Record.Source.System); records.add(record); diff --git a/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java b/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java index 779a7d3a..a5b6d19a 100644 --- a/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java +++ b/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java @@ -6,22 +6,52 @@ import java.util.Arrays; import java.util.Date; import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; /** * Created by yangsen on 2020/5/28 */ public class SystemDns implements Dns { - public List lookupInetAddress(String hostname) throws UnknownHostException { - InetAddress[] addressArray = InetAddress.getAllByName(hostname); - return Arrays.asList(addressArray); + private int timeout = 10; + + public SystemDns() { + } + + public SystemDns(int timeout) { + this.timeout = timeout; + } + + public List lookupInetAddress(final String hostname) throws UnknownHostException { + if (hostname == null) { + throw new UnknownHostException("hostname is null"); + } else { + try { + FutureTask> task = new FutureTask<>( + new Callable>() { + @Override + public List call() throws Exception { + return Arrays.asList(InetAddress.getAllByName(hostname)); + } + }); + new Thread(task).start(); + return task.get(timeout, TimeUnit.SECONDS); + } catch (Exception var4) { + UnknownHostException unknownHostException = + new UnknownHostException("Broken system behaviour for dns lookup of " + hostname); + unknownHostException.initCause(var4); + throw unknownHostException; + } + } } @Override public List lookup(String hostname) throws UnknownHostException { ArrayList addressList = new ArrayList<>(); - InetAddress[] addressArray = InetAddress.getAllByName(hostname); - for (InetAddress inetAddress : addressArray) { + List inetAddressList = lookupInetAddress(hostname); + for (InetAddress inetAddress : inetAddressList) { DnsNetworkAddress address = new DnsNetworkAddress(inetAddress.getHostName(), inetAddress.getHostAddress(), 120L, null, (new Date()).getTime()); addressList.add(address); } diff --git a/library/src/main/java/com/qiniu/android/http/request/httpclient/SystemHttpClient.java b/library/src/main/java/com/qiniu/android/http/request/httpclient/SystemHttpClient.java index 358ed1b5..33ab9fc0 100644 --- a/library/src/main/java/com/qiniu/android/http/request/httpclient/SystemHttpClient.java +++ b/library/src/main/java/com/qiniu/android/http/request/httpclient/SystemHttpClient.java @@ -9,6 +9,7 @@ import com.qiniu.android.http.request.Request; import com.qiniu.android.http.request.IRequestClient; import com.qiniu.android.http.metrics.UploadSingleRequestMetrics; +import com.qiniu.android.storage.GlobalConfiguration; import com.qiniu.android.utils.AsyncRun; import com.qiniu.android.utils.StringUtils; diff --git a/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java b/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java index b3cba8bc..58d3f31b 100644 --- a/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java +++ b/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java @@ -51,6 +51,16 @@ public class GlobalConfiguration { */ public int partialHostFrozenTime = 5 * 60; + /** + * 额外指定的 dns 预解析 server，当使用系统 dns 预解析失败时会使用 dnsServers 的顺序逐个进行预解析 + */ + public String[] dnsServers = new String[]{"114.114.114.114", "8.8.8.8", "119.29.29.29", "208.67.222.222", "223.5.5.5", "1.1.1.1"}; + + /** + * 额外指定的 {@link GlobalConfiguration#dnsServers} 在 dns 预取时的超时时间 + * 单位：秒 + */ + public int dnsServerResolveTimeout = 2; /** * 网络连接状态检测使用的connectCheckURLStrings，网络链接状态检测可能会影响重试机制，启动网络连接状态检测有助于提高上传可用性。 From 68fdd88161cba48142f78d7feca231cafd332ff7 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 14 Jul 2021 12:19:54 +0800 Subject: [PATCH 04/20] change dns prefetch --- .../com/qiniu/android/http/dns/HappyDns.java | 83 ++++++-------- .../com/qiniu/android/http/dns/HttpDns.java | 45 ++++++++ .../qiniu/android/http/dns/ServersDns.java | 108 ++++++++++++++++++ .../com/qiniu/android/http/dns/SystemDns.java | 6 +- 4 files changed, 191 insertions(+), 51 deletions(-) create mode 100644 library/src/main/java/com/qiniu/android/http/dns/HttpDns.java create mode 100644 library/src/main/java/com/qiniu/android/http/dns/ServersDns.java diff --git a/library/src/main/java/com/qiniu/android/http/dns/HappyDns.java b/library/src/main/java/com/qiniu/android/http/dns/HappyDns.java index a014dbb9..5c620ecf 100644 --- a/library/src/main/java/com/qiniu/android/http/dns/HappyDns.java +++ b/library/src/main/java/com/qiniu/android/http/dns/HappyDns.java @@ -20,76 +20,63 @@ */ public class HappyDns implements Dns { - private DnsManager dnsManager; + private SystemDns systemDns; + private ServersDns serversDns; + private HttpDns httpDns; + private DnsQueryErrorHandler errorHandler; public HappyDns(){ int dnsTimeout = GlobalConfiguration.getInstance().dnsServerResolveTimeout; - IResolver[] resolvers = new IResolver[3]; - resolvers[0] = new SystemResolver(dnsTimeout); - resolvers[1] = new DnsCustomServersResolver(GlobalConfiguration.getInstance().dnsServers, dnsTimeout); - resolvers[2] = new DnspodFree("119.29.29.29", dnsTimeout); - - dnsManager = new DnsManager(NetworkInfo.normal, resolvers); + systemDns = new SystemDns(dnsTimeout); + serversDns = new ServersDns(GlobalConfiguration.getInstance().dnsServers, dnsTimeout); + httpDns = new HttpDns(dnsTimeout); } void setQueryErrorHandler(DnsQueryErrorHandler handler){ - dnsManager.queryErrorHandler = handler; + errorHandler = handler; } @Override public List lookup(String hostname) throws UnknownHostException { - Domain domain = new Domain(hostname); List addressList = null; + + // 系统 dns try { - Record[] records = dnsManager.queryRecords(domain); - if (records != null && records.length > 0){ - addressList = new ArrayList<>(); - for (Record record : records) { - String source = ""; - if (record.source == Record.Source.System) { - source = "system"; - } else if (record.source == Record.Source.DnspodFree || - record.source == Record.Source.DnspodEnterprise) { - source = "httpdns"; - } else if (record.source == Record.Source.Unknown) { - source = "none"; - } else { - source = "customized"; - } - DnsNetworkAddress address = new DnsNetworkAddress(hostname, record.value, (long)record.ttl, source, record.timeStamp); - addressList.add(address); - } - } - } catch (IOException ignored) { + addressList = systemDns.lookup(hostname); + } catch (IOException e) { + handleDnsError(e, hostname); + } + if (addressList != null && addressList.size() > 0) { + return addressList; } - return addressList; - } - - private static class SystemResolver implements IResolver{ + // 指定 server ip dns + try { + addressList = serversDns.lookup(hostname); + } catch (IOException e) { + handleDnsError(e, hostname); + } + if (addressList != null && addressList.size() > 0) { + return addressList; + } - private int timeout; - SystemResolver(int timeout) { - this.timeout = timeout; + // http dns + try { + addressList = httpDns.lookup(hostname); + } catch (IOException e) { + handleDnsError(e, hostname); } - @Override - public Record[] resolve(Domain domain, NetworkInfo info) throws IOException { + return addressList; + } - long timestamp = Utils.currentTimestamp(); - int ttl = 120; - ArrayList records = new ArrayList<>(); - List inetAddresses = new SystemDns(timeout).lookupInetAddress(domain.domain); - for(InetAddress inetAddress : inetAddresses){ - Record record = new Record(inetAddress.getHostAddress(), Record.TYPE_A, ttl, timestamp, Record.Source.System); - records.add(record); - } - return records.toArray(new Record[0]); + private void handleDnsError(IOException e, String host) { + if (errorHandler != null) { + errorHandler.queryError(e, host); } } - interface DnsQueryErrorHandler extends DnsManager.QueryErrorHandler { } } diff --git a/library/src/main/java/com/qiniu/android/http/dns/HttpDns.java b/library/src/main/java/com/qiniu/android/http/dns/HttpDns.java new file mode 100644 index 00000000..f196f684 --- /dev/null +++ b/library/src/main/java/com/qiniu/android/http/dns/HttpDns.java @@ -0,0 +1,45 @@ +package com.qiniu.android.http.dns; + +import com.qiniu.android.dns.Domain; +import com.qiniu.android.dns.IResolver; +import com.qiniu.android.dns.Record; +import com.qiniu.android.dns.http.DnspodFree; + +import java.io.IOException; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +public class HttpDns implements Dns { + + private IResolver httpResolver; + + public HttpDns(int timeout) { + httpResolver = new DnspodFree("119.29.29.29", timeout); + } + + @Override + public List lookup(String hostname) throws UnknownHostException { + + Record[] records = null; + try { + records = httpResolver.resolve(new Domain(hostname), null); + } catch (IOException ignore) { + throw new UnknownHostException(ignore.toString()); + } + + if (records == null || records.length == 0) { + return null; + } + + ArrayList addressList = new ArrayList<>(); + for (Record record : records) { + DnsNetworkAddress address = new DnsNetworkAddress(hostname, record.value, record.timeStamp, "httpdns", (new Date()).getTime()); + addressList.add(address); + } + + return addressList; + } + +} diff --git a/library/src/main/java/com/qiniu/android/http/dns/ServersDns.java b/library/src/main/java/com/qiniu/android/http/dns/ServersDns.java new file mode 100644 index 00000000..8e55d4ca --- /dev/null +++ b/library/src/main/java/com/qiniu/android/http/dns/ServersDns.java @@ -0,0 +1,108 @@ +package com.qiniu.android.http.dns; + +import com.qiniu.android.dns.Domain; +import com.qiniu.android.dns.IResolver; +import com.qiniu.android.dns.Record; +import com.qiniu.android.dns.local.Resolver; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.List; + +public class ServersDns implements Dns { + + private ArrayList customServers = new ArrayList<>(); + + public ServersDns(String[] dnsServers, int timeout) { + if (timeout < 0 || timeout > 20) { + timeout = 2; + } + + for (String dnsServer : dnsServers) { + try { + InetAddress address = InetAddress.getByName(dnsServer); + customServers.add(new CustomServer(address, timeout)); + } catch (Exception ignore) { + } + } + } + + @Override + public List lookup(String hostname) throws UnknownHostException { + if (customServers.size() == 0) { + return null; + } + + List servers = getCustomServers(); + + String serverIP = null; + Record[] records = null; + for (CustomServer server : servers) { + try { + records = server.resolver.resolve(new Domain(hostname), null); + } catch (IOException ignore) { + } + + if (records != null && records.length > 0) { + serverIP = server.serverIP; + server.addReliability(); + break; + } + } + + ArrayList addressList = new ArrayList<>(); + for (Record record : records) { + DnsNetworkAddress address = new DnsNetworkAddress(hostname, record.value, record.timeStamp, serverIP, (new Date()).getTime()); + addressList.add(address); + } + + return addressList; + } + + private List getCustomServers() { + List servers = (List) customServers.clone(); + Comparator comparator = new Comparator() { + @Override + public int compare(CustomServer o1, CustomServer o2) { + if (o1.isMoreReliabilityThan(o2)) { + return -1; + } else { + return 1; + } + } + }; + + Collections.sort(servers, comparator); + return servers; + } + + private static class CustomServer { + + private String serverIP = null; + private int reliability = 0; + private IResolver resolver; + + CustomServer(InetAddress address, int timeout) { + this.resolver = new Resolver(address, timeout); + this.serverIP = address.getHostName(); + } + + private void addReliability() { + reliability += 1; + } + + private boolean isMoreReliabilityThan(CustomServer customServer) { + return reliability > customServer.reliability; + } + + @Override + public String toString() { + return "{" + "\"" + "reliability" + "\":" + reliability + "}"; + } + } +} diff --git a/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java b/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java index a5b6d19a..341ab5ae 100644 --- a/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java +++ b/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java @@ -24,7 +24,7 @@ public SystemDns(int timeout) { this.timeout = timeout; } - public List lookupInetAddress(final String hostname) throws UnknownHostException { + List lookupInetAddress(final String hostname) throws UnknownHostException { if (hostname == null) { throw new UnknownHostException("hostname is null"); } else { @@ -40,7 +40,7 @@ public List call() throws Exception { return task.get(timeout, TimeUnit.SECONDS); } catch (Exception var4) { UnknownHostException unknownHostException = - new UnknownHostException("Broken system behaviour for dns lookup of " + hostname); + new UnknownHostException("dns broken when lookup of " + hostname); unknownHostException.initCause(var4); throw unknownHostException; } @@ -52,7 +52,7 @@ public List lookup(String hostname) throws UnknownHostExcept ArrayList addressList = new ArrayList<>(); List inetAddressList = lookupInetAddress(hostname); for (InetAddress inetAddress : inetAddressList) { - DnsNetworkAddress address = new DnsNetworkAddress(inetAddress.getHostName(), inetAddress.getHostAddress(), 120L, null, (new Date()).getTime()); + DnsNetworkAddress address = new DnsNetworkAddress(inetAddress.getHostName(), inetAddress.getHostAddress(), 120L, "system", (new Date()).getTime()); addressList.add(address); } return addressList; From e30c9025b4c569cdbcfa3dbd88f8734b8a075bfb Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 14 Jul 2021 12:20:11 +0800 Subject: [PATCH 05/20] change dns prefetch --- .../http/dns/DnsCustomServersResolver.java | 91 ------------------- 1 file changed, 91 deletions(-) delete mode 100644 library/src/main/java/com/qiniu/android/http/dns/DnsCustomServersResolver.java diff --git a/library/src/main/java/com/qiniu/android/http/dns/DnsCustomServersResolver.java b/library/src/main/java/com/qiniu/android/http/dns/DnsCustomServersResolver.java deleted file mode 100644 index 1a398f53..00000000 --- a/library/src/main/java/com/qiniu/android/http/dns/DnsCustomServersResolver.java +++ /dev/null @@ -1,91 +0,0 @@ -package com.qiniu.android.http.dns; - -import com.qiniu.android.dns.Domain; -import com.qiniu.android.dns.IResolver; -import com.qiniu.android.dns.NetworkInfo; -import com.qiniu.android.dns.Record; -import com.qiniu.android.dns.local.Resolver; - -import java.io.IOException; -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; - -public class DnsCustomServersResolver implements IResolver { - - public ArrayList customServers = new ArrayList<>(); - - public DnsCustomServersResolver(String[] dnsServers, int timeout) { - if (timeout < 0 || timeout > 20) { - timeout = 2; - } - - for (String dnsServer : dnsServers) { - try { - InetAddress address = InetAddress.getByName(dnsServer); - customServers.add(new CustomServer(address, timeout)); - } catch (Exception ignore) { - } - - } - } - - @Override - public Record[] resolve(Domain domain, NetworkInfo info) throws IOException { - List servers = getCustomServers(); - - Record[] records = null; - for (CustomServer server : servers) { - records = server.resolver.resolve(domain, info); - if (records != null && records.length > 0) { - server.addReliability(); - break; - } - } - - return records; - } - - - public List getCustomServers() { - List servers = (List) customServers.clone(); - Comparator comparator = new Comparator() { - @Override - public int compare(CustomServer o1, CustomServer o2) { - if (o1.isMoreReliabilityThan(o2)) { - return -1; - } else { - return 1; - } - } - }; - - Collections.sort(servers, comparator); - return servers; - } - - public static class CustomServer { - - private int reliability = 0; - private IResolver resolver; - - CustomServer(InetAddress address, int timeout) { - this.resolver = new Resolver(address, timeout); - } - - public void addReliability() { - reliability += 1; - } - - boolean isMoreReliabilityThan(CustomServer customServer) { - return reliability > customServer.reliability; - } - - @Override - public String toString() { - return "{" + "\"" + "reliability" + "\":" + reliability + "}"; - } - } -} From 036aa6a5e707431940e2b7c3a9b4689f1eba09d9 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Wed, 14 Jul 2021 15:55:34 +0800 Subject: [PATCH 06/20] uplog add speed --- .../com/qiniu/android/collect/ReportItem.java | 3 ++ .../com/qiniu/android/http/dns/SystemDns.java | 2 +- .../android/http/metrics/UploadMetrics.java | 26 +++++++++++++++++ .../metrics/UploadRegionRequestMetrics.java | 26 ++++++++--------- .../metrics/UploadSingleRequestMetrics.java | 28 +++++++++--------- .../http/metrics/UploadTaskMetrics.java | 15 ++-------- .../http/request/HttpRegionRequest.java | 3 ++ .../http/request/HttpSingleRequest.java | 5 ++++ .../request/httpclient/SystemHttpClient.java | 12 +++++--- .../com/qiniu/android/storage/BaseUpload.java | 8 +++-- .../qiniu/android/storage/PartsUpload.java | 6 ++++ .../qiniu/android/storage/UploadManager.java | 29 ++++++++++++++----- .../java/com/qiniu/android/utils/Utils.java | 22 ++++++++++++++ 13 files changed, 131 insertions(+), 54 deletions(-) create mode 100644 library/src/main/java/com/qiniu/android/http/metrics/UploadMetrics.java diff --git a/library/src/main/java/com/qiniu/android/collect/ReportItem.java b/library/src/main/java/com/qiniu/android/collect/ReportItem.java index 285ec718..b764f2d3 100644 --- a/library/src/main/java/com/qiniu/android/collect/ReportItem.java +++ b/library/src/main/java/com/qiniu/android/collect/ReportItem.java @@ -82,6 +82,7 @@ public String toJson(){ public static final String RequestKeyPrefetchedBefore = "prefetched_before"; public static final String RequestKeyPrefetchedErrorMessage = "prefetched_error_message"; public static final String RequestKeyNetworkMeasuring = "network_measuring"; + public static final String RequestKeyPerceptiveSpeed = "perceptive_speed"; // 分块上传统计⽇志 public static final String BlockKeyLogType = "log_type"; @@ -102,6 +103,7 @@ public String toJson(){ public static final String BlockKeyOsVersion = "os_version"; public static final String BlockKeySDKName = "sdk_name"; public static final String BlockKeySDKVersion = "sdk_version"; + public static final String BlockKeyPerceptiveSpeed = "perceptive_speed"; // 上传质量统计 @@ -121,6 +123,7 @@ public String toJson(){ public static final String QualityKeyOsVersion = "os_version"; public static final String QualityKeySDKName = "sdk_name"; public static final String QualityKeySDKVersion = "sdk_version"; + public static final String QualityKeyPerceptiveSpeed = "perceptive_speed"; public static String requestReportStatusCode(ResponseInfo responseInfo){ if (responseInfo == null){ diff --git a/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java b/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java index 341ab5ae..f0d59612 100644 --- a/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java +++ b/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java @@ -24,7 +24,7 @@ public SystemDns(int timeout) { this.timeout = timeout; } - List lookupInetAddress(final String hostname) throws UnknownHostException { + public List lookupInetAddress(final String hostname) throws UnknownHostException { if (hostname == null) { throw new UnknownHostException("hostname is null"); } else { diff --git a/library/src/main/java/com/qiniu/android/http/metrics/UploadMetrics.java b/library/src/main/java/com/qiniu/android/http/metrics/UploadMetrics.java new file mode 100644 index 00000000..45e45388 --- /dev/null +++ b/library/src/main/java/com/qiniu/android/http/metrics/UploadMetrics.java @@ -0,0 +1,26 @@ +package com.qiniu.android.http.metrics; + +import com.qiniu.android.utils.Utils; + +import java.util.Date; + +public class UploadMetrics { + + protected Date startDate = null; + protected Date endDate = null; + + public void start() { + startDate = new Date(); + } + + public void end() { + endDate = new Date(); + } + + public long totalElapsedTime(){ + if (startDate == null || endDate == null) { + return 0; + } + return Utils.dateDuration(startDate, endDate); + } +} diff --git a/library/src/main/java/com/qiniu/android/http/metrics/UploadRegionRequestMetrics.java b/library/src/main/java/com/qiniu/android/http/metrics/UploadRegionRequestMetrics.java index 051d48ad..5d2b9fcf 100644 --- a/library/src/main/java/com/qiniu/android/http/metrics/UploadRegionRequestMetrics.java +++ b/library/src/main/java/com/qiniu/android/http/metrics/UploadRegionRequestMetrics.java @@ -1,32 +1,22 @@ package com.qiniu.android.http.metrics; import com.qiniu.android.http.request.IUploadRegion; +import com.qiniu.android.utils.Utils; +import java.util.Date; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; -public class UploadRegionRequestMetrics { +public class UploadRegionRequestMetrics extends UploadMetrics { public final IUploadRegion region; + private List metricsList = new CopyOnWriteArrayList<>(); public UploadRegionRequestMetrics(IUploadRegion region) { this.region = region; } - public long totalElapsedTime(){ - if (metricsList.size() == 0){ - return 0l; - } - long time = 0; - for (UploadSingleRequestMetrics metrics : metricsList){ - if (metrics != null){ - time += metrics.totalElapsedTime(); - } - } - return time; - } - public Integer requestCount(){ return metricsList.size(); } @@ -66,6 +56,14 @@ public void addMetrics(UploadRegionRequestMetrics metrics){ String thisRegionId = metrics.region.getZoneInfo().getRegionId(); String metricsRegionId = metrics.region.getZoneInfo().getRegionId(); if (thisRegionId.equals(metricsRegionId)){ + // 拼接开始和结束时间，开始时间要最老的，结束时间要最新的 + if (startDate != null && metrics.startDate != null && startDate.getTime() > metrics.startDate.getTime()) { + startDate = metrics.startDate; + } + if (endDate != null && metrics.endDate != null && endDate.getTime() < metrics.endDate.getTime()) { + endDate = metrics.endDate; + } + addMetricsList(metrics.metricsList); } } diff --git a/library/src/main/java/com/qiniu/android/http/metrics/UploadSingleRequestMetrics.java b/library/src/main/java/com/qiniu/android/http/metrics/UploadSingleRequestMetrics.java index 78864716..7c218946 100644 --- a/library/src/main/java/com/qiniu/android/http/metrics/UploadSingleRequestMetrics.java +++ b/library/src/main/java/com/qiniu/android/http/metrics/UploadSingleRequestMetrics.java @@ -2,12 +2,13 @@ import com.qiniu.android.http.ResponseInfo; import com.qiniu.android.http.request.Request; +import com.qiniu.android.utils.Utils; import org.json.JSONObject; import java.util.Date; -public class UploadSingleRequestMetrics { +public class UploadSingleRequestMetrics extends UploadMetrics { // 请求的 httpVersion public String httpVersion; @@ -21,9 +22,6 @@ public class UploadSingleRequestMetrics { public String clientName; public String clientVersion; - public Date startDate; - public Date endDate; - public Date domainLookupStartDate; public Date domainLookupEndDate; @@ -50,10 +48,6 @@ public class UploadSingleRequestMetrics { public String remoteAddress; public Integer remotePort; - - public long totalElapsedTime(){ - return time(startDate, endDate); - } public long totalDnsTime(){ return time(domainLookupStartDate, domainLookupEndDate); } @@ -95,6 +89,7 @@ public long totalBytes(){ } return (headerLength + bodyLength); } + public Long bytesSend(){ long totalBytes = totalBytes(); long bytesSend = countOfRequestHeaderBytesSent + countOfRequestBodyBytesSent; @@ -104,12 +99,19 @@ public Long bytesSend(){ return bytesSend; } + public Long bytesReceive(){ + long bytesReceive = countOfResponseHeaderBytesReceived + countOfResponseBodyBytesReceived; + if (bytesReceive < 0){ + bytesReceive = 0; + } + return bytesReceive; + } + + public Long perceptiveSpeed() { + return Utils.calculateSpeed(bytesSend() + bytesReceive(), totalElapsedTime()); + } private long time(Date startDate, Date endDate){ - if (startDate != null && endDate != null){ - return (endDate.getTime() - startDate.getTime()); - } else { - return 0l; - } + return Utils.dateDuration(startDate, endDate); } } diff --git a/library/src/main/java/com/qiniu/android/http/metrics/UploadTaskMetrics.java b/library/src/main/java/com/qiniu/android/http/metrics/UploadTaskMetrics.java index 8cf02966..52fc50ae 100644 --- a/library/src/main/java/com/qiniu/android/http/metrics/UploadTaskMetrics.java +++ b/library/src/main/java/com/qiniu/android/http/metrics/UploadTaskMetrics.java @@ -4,10 +4,11 @@ import com.qiniu.android.http.request.IUploadRegion; import java.util.ArrayList; +import java.util.Date; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -public class UploadTaskMetrics { +public class UploadTaskMetrics extends UploadMetrics { public ArrayList regions; private Map metricsInfo; @@ -17,18 +18,6 @@ public UploadTaskMetrics(ArrayList regions) { this.metricsInfo = new ConcurrentHashMap<>(); } - - public long totalElapsedTime(){ - long time = 0; - for (String key : metricsInfo.keySet()){ - UploadRegionRequestMetrics metrics = metricsInfo.get(key); - if (metrics != null){ - time += metrics.totalElapsedTime(); - } - } - return time; - } - public Long requestCount(){ long count = 0; for (String key : metricsInfo.keySet()){ diff --git a/library/src/main/java/com/qiniu/android/http/request/HttpRegionRequest.java b/library/src/main/java/com/qiniu/android/http/request/HttpRegionRequest.java index a7dc33d9..7c31eac7 100644 --- a/library/src/main/java/com/qiniu/android/http/request/HttpRegionRequest.java +++ b/library/src/main/java/com/qiniu/android/http/request/HttpRegionRequest.java @@ -51,6 +51,7 @@ void get(String action, RequestShouldRetryHandler shouldRetryHandler, RequestCompleteHandler completeHandler){ requestMetrics = new UploadRegionRequestMetrics(region); + requestMetrics.start(); performRequest(getNextServer(null), action, isAsync, null, header, "GET", shouldRetryHandler, null, completeHandler); } @@ -62,6 +63,7 @@ void post(String action, RequestProgressHandler progressHandler, RequestCompleteHandler completeHandler){ requestMetrics = new UploadRegionRequestMetrics(region); + requestMetrics.start(); performRequest(getNextServer(null), action, isAsync, data, header, "POST", shouldRetryHandler, progressHandler, completeHandler); } @@ -145,6 +147,7 @@ private void completeAction(ResponseInfo responseInfo, JSONObject response, RequestCompleteHandler completeHandler){ + requestMetrics.end(); singleRequest = null; if (completeHandler != null){ completeHandler.complete(responseInfo, requestMetrics, response); diff --git a/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java b/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java index 8601b3b1..60d14d8a 100644 --- a/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java +++ b/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java @@ -253,6 +253,11 @@ private void reportRequest(ResponseInfo responseInfo, item.setReport(networkMeasuring, ReportItem.RequestKeyNetworkMeasuring); } + // 统计当前请求上传速度 / 总耗时 + if (responseInfo.isOK()) { + item.setReport(requestMetrics.perceptiveSpeed(), ReportItem.RequestKeyPerceptiveSpeed); + } + item.setReport(requestMetrics.httpVersion, ReportItem.RequestKeyHttpVersion); UploadInfoReporter.getInstance().report(item, token.token); diff --git a/library/src/main/java/com/qiniu/android/http/request/httpclient/SystemHttpClient.java b/library/src/main/java/com/qiniu/android/http/request/httpclient/SystemHttpClient.java index 33ab9fc0..62de3176 100644 --- a/library/src/main/java/com/qiniu/android/http/request/httpclient/SystemHttpClient.java +++ b/library/src/main/java/com/qiniu/android/http/request/httpclient/SystemHttpClient.java @@ -265,7 +265,7 @@ private EventListener createEventLister() { return new EventListener() { @Override public void callStart(Call call) { - metrics.startDate = new Date(); + metrics.start(); } @Override @@ -360,7 +360,10 @@ public void responseHeadersStart(Call call) { @Override public void responseHeadersEnd(Call call, Response response) { - + Headers headers = response.headers(); + if (headers != null && headers.byteCount() > 0) { + metrics.countOfResponseHeaderBytesReceived = headers.byteCount(); + } } @Override @@ -370,6 +373,7 @@ public void responseBodyStart(Call call) { @Override public void responseBodyEnd(Call call, long byteCount) { metrics.responseEndDate = new Date(); + metrics.countOfResponseBodyBytesReceived = byteCount; } public void responseFailed(Call call, IOException ioe) { @@ -378,12 +382,12 @@ public void responseFailed(Call call, IOException ioe) { @Override public void callEnd(Call call) { - metrics.endDate = new Date(); + metrics.end(); } @Override public void callFailed(Call call, IOException ioe) { - metrics.endDate = new Date(); + metrics.end(); } }; } diff --git a/library/src/main/java/com/qiniu/android/storage/BaseUpload.java b/library/src/main/java/com/qiniu/android/storage/BaseUpload.java index 42a03288..dc9885ef 100644 --- a/library/src/main/java/com/qiniu/android/storage/BaseUpload.java +++ b/library/src/main/java/com/qiniu/android/storage/BaseUpload.java @@ -84,6 +84,8 @@ protected void initData() { @Override public void run() { + metrics.start(); + config.zone.preQuery(token, new Zone.QueryHandler() { @Override public void complete(int code, ResponseInfo responseInfo, UploadRegionRequestMetrics requestMetrics) { @@ -126,11 +128,13 @@ protected boolean switchRegionAndUpload() { return isSwitched; } - protected void completeAction(ResponseInfo responseInfo, - JSONObject response) { + protected void completeAction(ResponseInfo responseInfo, JSONObject response) { + if (currentRegionRequestMetrics != null && metrics != null) { + metrics.end(); metrics.addMetrics(currentRegionRequestMetrics); } + if (completionHandler != null) { completionHandler.complete(responseInfo, key, metrics, response); } diff --git a/library/src/main/java/com/qiniu/android/storage/PartsUpload.java b/library/src/main/java/com/qiniu/android/storage/PartsUpload.java index 284c4be1..2082bee1 100644 --- a/library/src/main/java/com/qiniu/android/storage/PartsUpload.java +++ b/library/src/main/java/com/qiniu/android/storage/PartsUpload.java @@ -287,6 +287,12 @@ private void reportBlock() { item.setReport(metrics.bytesSend(), ReportItem.BlockKeyBytesSent); item.setReport(uploadPerformer.recoveredFrom, ReportItem.BlockKeyRecoveredFrom); item.setReport(uploadSource.getSize(), ReportItem.BlockKeyFileSize); + + // 统计当前 region 上传速度 文件大小 / 总耗时 + if (uploadDataErrorResponseInfo == null && uploadSource.getSize() > 0 && metrics.totalElapsedTime() > 0) { + item.setReport(Utils.calculateSpeed(uploadSource.getSize(), metrics.totalElapsedTime()), ReportItem.BlockKeyPerceptiveSpeed); + } + item.setReport(Utils.getCurrentProcessID(), ReportItem.BlockKeyPid); item.setReport(Utils.getCurrentThreadID(), ReportItem.BlockKeyTid); diff --git a/library/src/main/java/com/qiniu/android/storage/UploadManager.java b/library/src/main/java/com/qiniu/android/storage/UploadManager.java index 9f22f41c..2f158b9b 100644 --- a/library/src/main/java/com/qiniu/android/storage/UploadManager.java +++ b/library/src/main/java/com/qiniu/android/storage/UploadManager.java @@ -338,7 +338,7 @@ private void putData(final byte[] data, final UpToken t = UpToken.parse(token); if (t == null || !t.isValid()) { ResponseInfo info = ResponseInfo.invalidToken("invalid token"); - completeAction(token, key, info, null, null, completionHandler); + completeAction(token, key, data, info, null, null, completionHandler); return; } @@ -347,7 +347,7 @@ private void putData(final byte[] data, BaseUpload.UpTaskCompletionHandler completionHandlerP = new BaseUpload.UpTaskCompletionHandler() { @Override public void complete(ResponseInfo responseInfo, String key, UploadTaskMetrics requestMetrics, JSONObject response) { - completeAction(token, key, responseInfo, response, requestMetrics, completionHandler); + completeAction(token, key, data, responseInfo, response, requestMetrics, completionHandler); } }; final FormUpload up = new FormUpload(data, key, fileName, t, option, config, completionHandlerP); @@ -367,7 +367,7 @@ private void putSource(final UploadSource source, final UpToken t = UpToken.parse(token); if (t == null || !t.isValid()) { ResponseInfo info = ResponseInfo.invalidToken("invalid token"); - completeAction(token, key, info, null, null, completionHandler); + completeAction(token, key, source, info, null, null, completionHandler); return; } @@ -386,7 +386,7 @@ private void putSource(final UploadSource source, if (errorInfo == null) { putData(data, source.getFileName(), key, token, option, completionHandler); } else { - completeAction(token, key, errorInfo, null, null, completionHandler); + completeAction(token, key, source, errorInfo, null, null, completionHandler); } return; } @@ -399,7 +399,7 @@ private void putSource(final UploadSource source, BaseUpload.UpTaskCompletionHandler completionHandlerP = new BaseUpload.UpTaskCompletionHandler() { @Override public void complete(ResponseInfo responseInfo, String key, UploadTaskMetrics requestMetrics, JSONObject response) { - completeAction(token, key, responseInfo, response, requestMetrics, completionHandler); + completeAction(token, key, source, responseInfo, response, requestMetrics, completionHandler); } }; if (config.useConcurrentResumeUpload) { @@ -432,7 +432,7 @@ private boolean checkAndNotifyError(String key, responseInfo = ResponseInfo.invalidToken("no token"); } if (responseInfo != null) { - completeAction(token, key, responseInfo, responseInfo.response, null, completionHandler); + completeAction(token, key, null, responseInfo, responseInfo.response, null, completionHandler); return true; } else { return false; @@ -441,12 +441,13 @@ private boolean checkAndNotifyError(String key, private void completeAction(final String token, final String key, + final Object source, final ResponseInfo responseInfo, final JSONObject response, final UploadTaskMetrics taskMetrics, final UpCompletionHandler completionHandler) { - reportQuality(key, responseInfo, taskMetrics, token); + reportQuality(key, source, responseInfo, taskMetrics, token); if (completionHandler != null) { final Wait wait = new Wait(); AsyncRun.runInMain(new Runnable() { @@ -461,6 +462,7 @@ public void run() { } private void reportQuality(String key, + Object source, ResponseInfo responseInfo, UploadTaskMetrics taskMetrics, String token) { @@ -495,6 +497,19 @@ private void reportQuality(String key, item.setReport(errorDesc, ReportItem.QualityKeyErrorDescription); } + // 统计当前文件上传速度，也即用户感知速度： 总文件大小 / 总耗时 + if (source != null && responseInfo.isOK() && taskMetrics.totalElapsedTime() > 0) { + long fileSize = 0; + if (source instanceof UploadSource) { + fileSize = ((UploadSource) source).getSize(); + } else if (source instanceof byte[]) { + fileSize = ((byte[]) source).length; + } + if (fileSize > 0) { + item.setReport(Utils.calculateSpeed(fileSize, taskMetrics.totalElapsedTime()), ReportItem.QualityKeyPerceptiveSpeed); + } + } + UploadInfoReporter.getInstance().report(item, token); } } diff --git a/library/src/main/java/com/qiniu/android/utils/Utils.java b/library/src/main/java/com/qiniu/android/utils/Utils.java index 07e0d1df..af2128b9 100644 --- a/library/src/main/java/com/qiniu/android/utils/Utils.java +++ b/library/src/main/java/com/qiniu/android/utils/Utils.java @@ -72,6 +72,28 @@ public static long currentTimestamp() { return new Date().getTime(); } + /// 两个时间的时间段 单位：毫秒 + public static long dateDuration(Date startDate, Date endDate){ + if (startDate != null && endDate != null){ + return (endDate.getTime() - startDate.getTime()); + } else { + return 0l; + } + } + + /** + * 计算 上传 或 下载 速度 单位：B/s + * @param bytes 单位： B + * @param totalTime 单位：ms + * @return 速度 + */ + public static Long calculateSpeed(Long bytes, Long totalTime){ + if (bytes == null || bytes < 0 || totalTime == null || totalTime == 0) { + return null; + } + return bytes * 1000 / totalTime; + } + public static String sdkDirectory() { Context context = ContextGetter.applicationContext(); if (context == null) { From 31e918e11a424d0e32b5151edf2ed25887bf7e13 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 6 Aug 2021 17:15:23 +0800 Subject: [PATCH 07/20] modify block metrics start date and end date logic --- library/library.iml | 8 ++++---- .../android/http/request/HttpSingleRequest.java | 2 +- .../java/com/qiniu/android/storage/BaseUpload.java | 14 ++++++++++++-- .../java/com/qiniu/android/storage/FormUpload.java | 1 + .../com/qiniu/android/storage/PartsUpload.java | 2 ++ 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/library/library.iml b/library/library.iml index 244cd7b6..bbb68de1 100644 --- a/library/library.iml +++ b/library/library.iml @@ -31,16 +31,16 @@ - - - + + + - + diff --git a/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java b/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java index 60d14d8a..5bb02445 100644 --- a/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java +++ b/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java @@ -235,7 +235,7 @@ private void reportRequest(ResponseInfo responseInfo, item.setReport(server.getSource(), ReportItem.RequestKeyPrefetchedDnsSource); if (server.getIpPrefetchedTime() != null) { - Long prefetchTime = server.getIpPrefetchedTime() - currentTimestamp; + Long prefetchTime = (long)(currentTimestamp*0.0001) - server.getIpPrefetchedTime(); item.setReport(prefetchTime, ReportItem.RequestKeyPrefetchedBefore); } item.setReport(DnsPrefetcher.getInstance().lastPrefetchErrorMessage, ReportItem.RequestKeyPrefetchedErrorMessage); diff --git a/library/src/main/java/com/qiniu/android/storage/BaseUpload.java b/library/src/main/java/com/qiniu/android/storage/BaseUpload.java index dc9885ef..8ebf4503 100644 --- a/library/src/main/java/com/qiniu/android/storage/BaseUpload.java +++ b/library/src/main/java/com/qiniu/android/storage/BaseUpload.java @@ -114,13 +114,18 @@ protected int prepareToUpload() { return ret; } - protected abstract void startToUpload(); + protected void startToUpload() { + currentRegionRequestMetrics = new UploadRegionRequestMetrics(getCurrentRegion()); + currentRegionRequestMetrics.start(); + } protected boolean switchRegionAndUpload() { if (currentRegionRequestMetrics != null) { + currentRegionRequestMetrics.end(); metrics.addMetrics(currentRegionRequestMetrics); currentRegionRequestMetrics = null; } + boolean isSwitched = switchRegion(); if (isSwitched) { startToUpload(); @@ -129,9 +134,14 @@ protected boolean switchRegionAndUpload() { } protected void completeAction(ResponseInfo responseInfo, JSONObject response) { + if (metrics != null) { + metrics.end(); + } + if (currentRegionRequestMetrics != null) { + currentRegionRequestMetrics.end(); + } if (currentRegionRequestMetrics != null && metrics != null) { - metrics.end(); metrics.addMetrics(currentRegionRequestMetrics); } diff --git a/library/src/main/java/com/qiniu/android/storage/FormUpload.java b/library/src/main/java/com/qiniu/android/storage/FormUpload.java index 0c0a6a58..389959d5 100644 --- a/library/src/main/java/com/qiniu/android/storage/FormUpload.java +++ b/library/src/main/java/com/qiniu/android/storage/FormUpload.java @@ -28,6 +28,7 @@ protected FormUpload(byte[] data, @Override protected void startToUpload() { + super.startToUpload(); LogUtil.i("key:" + StringUtils.toNonnullString(key) + " form上传"); diff --git a/library/src/main/java/com/qiniu/android/storage/PartsUpload.java b/library/src/main/java/com/qiniu/android/storage/PartsUpload.java index 2082bee1..ffea7bec 100644 --- a/library/src/main/java/com/qiniu/android/storage/PartsUpload.java +++ b/library/src/main/java/com/qiniu/android/storage/PartsUpload.java @@ -115,6 +115,8 @@ protected boolean switchRegionAndUpload() { @Override protected void startToUpload() { + super.startToUpload(); + uploadDataErrorResponse = null; uploadDataErrorResponseInfo = null; From d6e71bc84482e1f1f1081cee556a750d3089d457 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 6 Aug 2021 17:56:28 +0800 Subject: [PATCH 08/20] add dns max ttl --- .../android/http/dns/DnsNetworkAddress.java | 19 +++ .../qiniu/android/http/dns/DnsPrefetcher.java | 117 ++++++++++-------- .../android/storage/GlobalConfiguration.java | 9 +- 3 files changed, 91 insertions(+), 54 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/http/dns/DnsNetworkAddress.java b/library/src/main/java/com/qiniu/android/http/dns/DnsNetworkAddress.java index 22781b07..212a4038 100644 --- a/library/src/main/java/com/qiniu/android/http/dns/DnsNetworkAddress.java +++ b/library/src/main/java/com/qiniu/android/http/dns/DnsNetworkAddress.java @@ -1,5 +1,8 @@ package com.qiniu.android.http.dns; +import com.qiniu.android.storage.GlobalConfiguration; +import com.qiniu.android.utils.Utils; + import org.json.JSONException; import org.json.JSONObject; @@ -93,4 +96,20 @@ public String getSourceValue() { public Long getTimestampValue() { return timestampValue; } + + boolean isValid() { + if (timestampValue == null || ipValue == null || ipValue.length() == 0) { + return false; + } + int maxTTL = GlobalConfiguration.getInstance().dnsCacheMaxTTL; + return (Utils.currentTimestamp() / 1000) < timestampValue + maxTTL; + } + + boolean needRefresh() { + if (timestampValue == null || ttlValue == null || ipValue == null || ipValue.length() == 0) { + return false; + } + int ttl = ttlValue.intValue(); + return (Utils.currentTimestamp() / 1000) > timestampValue + ttl; + } } \ No newline at end of file diff --git a/library/src/main/java/com/qiniu/android/http/dns/DnsPrefetcher.java b/library/src/main/java/com/qiniu/android/http/dns/DnsPrefetcher.java index b34d0b14..458367df 100644 --- a/library/src/main/java/com/qiniu/android/http/dns/DnsPrefetcher.java +++ b/library/src/main/java/com/qiniu/android/http/dns/DnsPrefetcher.java @@ -33,7 +33,8 @@ public class DnsPrefetcher { private final HappyDns happyDns = new HappyDns(); private final static DnsPrefetcher dnsPrefetcher = new DnsPrefetcher(); - private DnsPrefetcher(){ + + private DnsPrefetcher() { happyDns.setQueryErrorHandler(new HappyDns.DnsQueryErrorHandler() { @Override public void queryError(Exception e, String host) { @@ -42,13 +43,13 @@ public void queryError(Exception e, String host) { }); } - public static DnsPrefetcher getInstance(){ + public static DnsPrefetcher getInstance() { return dnsPrefetcher; } public String lastPrefetchErrorMessage; - public boolean recoverCache(){ + public boolean recoverCache() { DnsCacheFile recorder = null; try { @@ -58,23 +59,23 @@ public boolean recoverCache(){ } String localIp = AndroidNetwork.getHostIP(); - if (localIp == null || localIp.length() == 0){ + if (localIp == null || localIp.length() == 0) { return true; } byte[] data = recorder.get(localIp); - if (data == null){ + if (data == null) { return true; } return recoverDnsCache(data); } - public void localFetch(){ + public void localFetch() { addPreFetchHosts(getLocalPreHost()); } - public boolean checkAndPrefetchDnsIfNeed(Zone currentZone, UpToken token){ + public boolean checkAndPrefetchDnsIfNeed(Zone currentZone, UpToken token) { return addPreFetchHosts(getCurrentZoneHosts(currentZone, token)); } @@ -102,37 +103,40 @@ public boolean addPreFetchHosts(String[] hosts) { } } - public void invalidNetworkAddress(IDnsNetworkAddress address){ - if (address == null || address.getHostValue() == null){ + public void invalidNetworkAddress(IDnsNetworkAddress address) { + if (address == null || address.getHostValue() == null) { return; } String host = address.getHostValue(); List addressList = addressDictionary.get(host); ArrayList addressListNew = new ArrayList<>(); - for (IDnsNetworkAddress addressP : addressList){ - if (!addressP.getIpValue().equals(addressP.getIpValue())){ + for (IDnsNetworkAddress addressP : addressList) { + if (!addressP.getIpValue().equals(addressP.getIpValue())) { addressListNew.add(addressP); } } addressDictionary.put(host, addressListNew); } - public List getInetAddressByHost(String host){ - if (!isDnsOpen()){ + public List getInetAddressByHost(String host) { + if (!isDnsOpen()) { return null; } List addressList = addressDictionary.get(host); - if (addressList != null && addressList.size() > 0){ - return addressList; - } else { - return null; + if (addressList != null && addressList.size() > 0) { + DnsNetworkAddress firstAddress = (DnsNetworkAddress) addressList.get(0); + if (firstAddress.isValid()) { + return addressList; + } } + + return null; } - public void checkWhetherCachedDnsValid(){ - if (!prepareToPreFetch()){ + public void checkWhetherCachedDnsValid() { + if (!prepareToPreFetch()) { return; } @@ -146,17 +150,17 @@ public void checkWhetherCachedDnsValid(){ } - private synchronized boolean prepareToPreFetch(){ - if (!isDnsOpen()){ + private synchronized boolean prepareToPreFetch() { + if (!isDnsOpen()) { return false; } - if (isPrefetching()){ + if (isPrefetching()) { return false; } String localIp = AndroidNetwork.getHostIP(); - if (localIp == null || getDnsCacheInfo() == null || !(localIp.equals(getDnsCacheInfo().getLocalIp()))){ + if (localIp == null || getDnsCacheInfo() == null || !(localIp.equals(getDnsCacheInfo().getLocalIp()))) { clearPreHosts(); } @@ -164,59 +168,62 @@ private synchronized boolean prepareToPreFetch(){ return true; } - private void endPreFetch(){ + private void endPreFetch() { setPrefetching(false); } - private void preFetchHosts(String[] fetchHosts){ + private void preFetchHosts(String[] fetchHosts) { String[] nextFetchHosts = fetchHosts; nextFetchHosts = preFetchHosts(nextFetchHosts, GlobalConfiguration.getInstance().dns); nextFetchHosts = preFetchHosts(nextFetchHosts, happyDns); } - private String[] preFetchHosts(String[] preHosts, Dns dns){ - if (preHosts == null || preHosts.length == 0){ + private String[] preFetchHosts(String[] preHosts, Dns dns) { + if (preHosts == null || preHosts.length == 0) { return null; } - if (dns == null){ + if (dns == null) { return preHosts; } ArrayList failHosts = new ArrayList<>(); - for (String host : preHosts){ + for (String host : preHosts) { int rePreNum = 0; boolean isSuccess = false; - while (rePreNum < GlobalConfiguration.getInstance().dnsRepreHostNum){ - if (preFetchHost(host, dns)){ + while (rePreNum < GlobalConfiguration.getInstance().dnsRepreHostNum) { + if (preFetchHost(host, dns)) { isSuccess = true; break; } rePreNum += 1; } - if (!isSuccess){ + if (!isSuccess) { failHosts.add(host); } } return failHosts.toArray(new String[0]); - } + } - private boolean preFetchHost(String preHost, Dns dns){ - if (preHost == null || preHost.length() == 0){ + private boolean preFetchHost(String preHost, Dns dns) { + if (preHost == null || preHost.length() == 0) { return false; } List preAddressList = addressDictionary.get(preHost); - if (preAddressList != null && preAddressList.size() > 0){ - return true; + if (preAddressList != null && preAddressList.size() > 0) { + DnsNetworkAddress firstAddress = (DnsNetworkAddress) preAddressList.get(0); + if (!firstAddress.needRefresh()) { + return true; + } } List addressList = new ArrayList<>(); try { List preIAddressList = dns.lookup(preHost); - if (preIAddressList != null && preIAddressList.size() > 0){ + if (preIAddressList != null && preIAddressList.size() > 0) { for (IDnsNetworkAddress preIAddress : preIAddressList) { DnsNetworkAddress address = new DnsNetworkAddress(preIAddress.getHostValue(), preIAddress.getIpValue(), @@ -226,8 +233,10 @@ private boolean preFetchHost(String preHost, Dns dns){ addressList.add(address); } } - } catch (UnknownHostException e) {} - if (addressList != null && addressList.size() > 0){ + } catch (UnknownHostException e) { + } + + if (addressList != null && addressList.size() > 0) { addressDictionary.put(preHost, addressList); return true; } else { @@ -235,10 +244,10 @@ private boolean preFetchHost(String preHost, Dns dns){ } } - private boolean recoverDnsCache(byte[] data){ + private boolean recoverDnsCache(byte[] data) { DnsCacheInfo dnsCacheInfo = DnsCacheInfo.createDnsCacheInfoByData(data); - if (dnsCacheInfo == null || dnsCacheInfo.getInfo() == null || dnsCacheInfo.getInfo().size() == 0){ + if (dnsCacheInfo == null || dnsCacheInfo.getInfo() == null || dnsCacheInfo.getInfo().size() == 0) { return false; } @@ -249,11 +258,11 @@ private boolean recoverDnsCache(byte[] data){ return false; } - private boolean recorderDnsCache(){ + private boolean recorderDnsCache() { String currentTime = Utils.currentTimestamp() + ""; String localIp = AndroidNetwork.getHostIP(); - if (localIp == null){ + if (localIp == null) { return false; } @@ -269,7 +278,7 @@ private boolean recorderDnsCache(){ setDnsCacheInfo(dnsCacheInfo); byte[] data = dnsCacheInfo.toJsonData(); - if (data == null){ + if (data == null) { return false; } recorder.set(dnsCacheInfo.cacheKey(), data); @@ -277,17 +286,17 @@ private boolean recorderDnsCache(){ return true; } - private void clearPreHosts(){ + private void clearPreHosts() { addressDictionary.clear(); } - private String[] getLocalPreHost(){ + private String[] getLocalPreHost() { return new String[]{Config.upLogURL}; } - private String[] getCurrentZoneHosts(Zone currentZone, UpToken token){ - if (currentZone == null || token == null){ + private String[] getCurrentZoneHosts(Zone currentZone, UpToken token) { + if (currentZone == null || token == null) { return null; } @@ -314,29 +323,30 @@ public void complete(int code, ResponseInfo responseInfo, UploadRegionRequestMet return autoHosts.toArray(new String[0]); } - private String[] getFixedZoneHosts(){ + private String[] getFixedZoneHosts() { ArrayList localHosts = new ArrayList<>(); FixedZone fixedZone = FixedZone.localsZoneInfo(); ZonesInfo zonesInfo = fixedZone.getZonesInfo(null); for (ZoneInfo zoneInfo : zonesInfo.zonesInfo) { - if (zoneInfo != null && zoneInfo.allHosts != null){ + if (zoneInfo != null && zoneInfo.allHosts != null) { localHosts.addAll(zoneInfo.allHosts); } } return localHosts.toArray(new String[0]); } - private String[] getCacheHosts(){ + private String[] getCacheHosts() { return addressDictionary.keySet().toArray(new String[0]); } - public boolean isDnsOpen(){ + public boolean isDnsOpen() { return GlobalConfiguration.getInstance().isDnsOpen; } public synchronized boolean isPrefetching() { return isPrefetching; } + private synchronized void setPrefetching(boolean isPrefetching) { this.isPrefetching = isPrefetching; } @@ -344,6 +354,7 @@ private synchronized void setPrefetching(boolean isPrefetching) { private synchronized DnsCacheInfo getDnsCacheInfo() { return dnsCacheInfo; } + private synchronized void setDnsCacheInfo(DnsCacheInfo dnsCacheInfo) { this.dnsCacheInfo = dnsCacheInfo; } diff --git a/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java b/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java index 58d3f31b..75240854 100644 --- a/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java +++ b/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java @@ -23,10 +23,17 @@ public class GlobalConfiguration { public int dnsRepreHostNum = 2; /** - * dns预取缓存时间 单位：秒 + * dns 预取, ip 默认有效时间 单位：秒 默认：120 + * 只有在 dns 预取未返回 ttl 时使用 */ public int dnsCacheTime = 120; + /** + * dns预取缓存最大有效时间 单位：秒 默认 1800 + * 当 dns 缓存 ip 过期并未刷新时，只要在 dnsCacheMaxTTL 时间内仍有效。 + */ + public int dnsCacheMaxTTL = 1800; + /** * 自定义DNS解析客户端host */ From 41cd66b56244babe3c30ac2705dfceea15f7a204 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 6 Aug 2021 18:22:01 +0800 Subject: [PATCH 09/20] connect checker add enable config --- .../http/connectCheck/ConnectChecker.java | 35 ++++++++++++++++++- .../http/request/HttpSingleRequest.java | 5 +++ .../android/storage/GlobalConfiguration.java | 5 +++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/library/src/main/java/com/qiniu/android/http/connectCheck/ConnectChecker.java b/library/src/main/java/com/qiniu/android/http/connectCheck/ConnectChecker.java index 8bd9e94a..1c6e08bd 100644 --- a/library/src/main/java/com/qiniu/android/http/connectCheck/ConnectChecker.java +++ b/library/src/main/java/com/qiniu/android/http/connectCheck/ConnectChecker.java @@ -12,8 +12,14 @@ import org.json.JSONObject; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + public class ConnectChecker { + private static ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); private static SingleFlight singleFlight = new SingleFlight<>(); public static boolean isConnected(UploadSingleRequestMetrics metrics) { @@ -106,13 +112,40 @@ public void complete(UploadSingleRequestMetrics metrics) { private static void checkHost(final String host, final CheckCompleteHandler completeHandler) { - Request request = new Request(host, Request.HttpMethodHEAD, null, null, GlobalConfiguration.getInstance().connectCheckTimeout); + final boolean[] hasCallback = {false}; + int timeout = GlobalConfiguration.getInstance().connectCheckTimeout; + + final UploadSingleRequestMetrics timeoutMetrics = new UploadSingleRequestMetrics(); + timeoutMetrics.start(); + executorService.schedule(new Callable() { + @Override + public Object call() throws Exception { + synchronized (this) { + if (hasCallback[0]) { + return null; + } + hasCallback[0] = true; + } + timeoutMetrics.end(); + completeHandler.complete(timeoutMetrics); + return null; + } + }, timeout, TimeUnit.SECONDS); + + Request request = new Request(host, Request.HttpMethodHEAD, null, null, timeout); SystemHttpClient client = new SystemHttpClient(); LogUtil.i("== checkHost:" + host); client.request(request, true, null, null, new IRequestClient.RequestClientCompleteHandler() { @Override public void complete(ResponseInfo responseInfo, UploadSingleRequestMetrics metrics, JSONObject response) { + synchronized (this) { + if (hasCallback[0]) { + return; + } + hasCallback[0] = true; + } + LogUtil.i("== checkHost:" + host + " responseInfo:" + responseInfo); completeHandler.complete(metrics); } diff --git a/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java b/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java index 5bb02445..393847cf 100644 --- a/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java +++ b/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java @@ -12,6 +12,7 @@ import com.qiniu.android.http.request.handler.RequestShouldRetryHandler; import com.qiniu.android.http.metrics.UploadSingleRequestMetrics; import com.qiniu.android.storage.Configuration; +import com.qiniu.android.storage.GlobalConfiguration; import com.qiniu.android.storage.UpToken; import com.qiniu.android.storage.UploadOptions; import com.qiniu.android.utils.LogUtil; @@ -141,6 +142,10 @@ public void complete(ResponseInfo responseInfo, UploadSingleRequestMetrics metri } private boolean shouldCheckConnect(ResponseInfo responseInfo) { + if (!GlobalConfiguration.getInstance().connectCheckEnable) { + return false; + } + return responseInfo != null && (responseInfo.statusCode == ResponseInfo.NetworkError || /* network error */ responseInfo.statusCode == -1001 || /* timeout */ diff --git a/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java b/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java index 75240854..49c435e2 100644 --- a/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java +++ b/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java @@ -81,6 +81,11 @@ public class GlobalConfiguration { */ public int connectCheckTimeout = 3; + /** + * 是否开启网络连接状态检测，默认：开启 + */ + public boolean connectCheckEnable = true; + private static GlobalConfiguration configuration = new GlobalConfiguration(); private GlobalConfiguration() { From 03638a65a5ed161014e667e0b7c3e2c26498788e Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 6 Aug 2021 18:29:01 +0800 Subject: [PATCH 10/20] optimize connect checker --- .../com/qiniu/android/http/connectCheck/ConnectChecker.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/http/connectCheck/ConnectChecker.java b/library/src/main/java/com/qiniu/android/http/connectCheck/ConnectChecker.java index 1c6e08bd..3e4cb2fe 100644 --- a/library/src/main/java/com/qiniu/android/http/connectCheck/ConnectChecker.java +++ b/library/src/main/java/com/qiniu/android/http/connectCheck/ConnectChecker.java @@ -87,9 +87,6 @@ public void complete(UploadSingleRequestMetrics metrics) { synchronized (checkStatus) { checkStatus.completeCount += 1; } - if (isHostConnected) { - checkStatus.isConnected = true; - } if (isHostConnected || checkStatus.completeCount == checkStatus.totalCount) { synchronized (checkStatus) { if (checkStatus.isCompleted) { @@ -161,7 +158,6 @@ private static class CheckStatus { private int totalCount = 0; private int completeCount = 0; private boolean isCompleted = false; - private boolean isConnected = false; } private static class CheckResult { From 829bdd850cae3a3f042766f3cb312c6470077c74 Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Fri, 6 Aug 2021 18:40:58 +0800 Subject: [PATCH 11/20] modify dns prefetch before --- library/src/main/java/com/qiniu/android/http/dns/SystemDns.java | 2 +- .../java/com/qiniu/android/http/request/HttpSingleRequest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java b/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java index f0d59612..617432fd 100644 --- a/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java +++ b/library/src/main/java/com/qiniu/android/http/dns/SystemDns.java @@ -52,7 +52,7 @@ public List lookup(String hostname) throws UnknownHostExcept ArrayList addressList = new ArrayList<>(); List inetAddressList = lookupInetAddress(hostname); for (InetAddress inetAddress : inetAddressList) { - DnsNetworkAddress address = new DnsNetworkAddress(inetAddress.getHostName(), inetAddress.getHostAddress(), 120L, "system", (new Date()).getTime()); + DnsNetworkAddress address = new DnsNetworkAddress(inetAddress.getHostName(), inetAddress.getHostAddress(), 120L, "system", (new Date()).getTime()/1000); addressList.add(address); } return addressList; diff --git a/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java b/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java index 393847cf..8dfd5b77 100644 --- a/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java +++ b/library/src/main/java/com/qiniu/android/http/request/HttpSingleRequest.java @@ -240,7 +240,7 @@ private void reportRequest(ResponseInfo responseInfo, item.setReport(server.getSource(), ReportItem.RequestKeyPrefetchedDnsSource); if (server.getIpPrefetchedTime() != null) { - Long prefetchTime = (long)(currentTimestamp*0.0001) - server.getIpPrefetchedTime(); + Long prefetchTime = currentTimestamp/1000 - server.getIpPrefetchedTime(); item.setReport(prefetchTime, ReportItem.RequestKeyPrefetchedBefore); } item.setReport(DnsPrefetcher.getInstance().lastPrefetchErrorMessage, ReportItem.RequestKeyPrefetchedErrorMessage); From 0b1a40883a348c2cc0ebc76a35eab5c6cb6e8cbc Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 9 Aug 2021 11:42:20 +0800 Subject: [PATCH 12/20] version to 8.3.2 & remove dns server --- CHANGELOG.md | 5 + README.md | 2 +- .../com/qiniu/android/common/Constants.java | 2 +- .../com/qiniu/android/http/dns/HappyDns.java | 22 +--- .../qiniu/android/http/dns/ServersDns.java | 108 ------------------ .../metrics/UploadSingleRequestMetrics.java | 25 ++-- .../com/qiniu/android/storage/BaseUpload.java | 3 - .../android/storage/GlobalConfiguration.java | 9 +- .../qiniu/android/storage/PartsUpload.java | 4 +- 9 files changed, 26 insertions(+), 154 deletions(-) delete mode 100644 library/src/main/java/com/qiniu/android/http/dns/ServersDns.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 80909975..69e2f542 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,9 @@ #Changelog +## 8.3.2(2021-08-09) +* dns 缓存增加最大有效时间配置 +* 网络状态检测可关闭 + + ## 8.3.1(2021-07-12) * 兼容支持 Android 4.x ( API level 14+ )，使用 Android4.x 对应 okhttp 版本请调整至 3.12.+ * 支持 okhttp 4.9 diff --git a/README.md b/README.md index 7be90f20..58869a32 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ https://github.com/qiniudemo/qiniu-lab-android | 7.0.7 | Android 2.2+ | android-async-http 1.4.8 | ### 注意 -* 推荐使用最新版：8.3.1 +* 推荐使用最新版：8.3.2 * AndroidNetwork.getMobileDbm()可以获取手机信号强度，需要如下权限(API>=18时生效) ``` diff --git a/library/src/main/java/com/qiniu/android/common/Constants.java b/library/src/main/java/com/qiniu/android/common/Constants.java index c548ccf8..cee2b817 100644 --- a/library/src/main/java/com/qiniu/android/common/Constants.java +++ b/library/src/main/java/com/qiniu/android/common/Constants.java @@ -2,7 +2,7 @@ public final class Constants { - public static final String VERSION = "8.3.1"; + public static final String VERSION = "8.3.2"; public static final String UTF_8 = "utf-8"; } diff --git a/library/src/main/java/com/qiniu/android/http/dns/HappyDns.java b/library/src/main/java/com/qiniu/android/http/dns/HappyDns.java index 5c620ecf..1042ba77 100644 --- a/library/src/main/java/com/qiniu/android/http/dns/HappyDns.java +++ b/library/src/main/java/com/qiniu/android/http/dns/HappyDns.java @@ -1,18 +1,10 @@ package com.qiniu.android.http.dns; import com.qiniu.android.dns.DnsManager; -import com.qiniu.android.dns.Domain; -import com.qiniu.android.dns.IResolver; -import com.qiniu.android.dns.NetworkInfo; -import com.qiniu.android.dns.Record; -import com.qiniu.android.dns.http.DnspodFree; import com.qiniu.android.storage.GlobalConfiguration; -import com.qiniu.android.utils.Utils; import java.io.IOException; -import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.ArrayList; import java.util.List; /** @@ -21,15 +13,13 @@ public class HappyDns implements Dns { private SystemDns systemDns; - private ServersDns serversDns; private HttpDns httpDns; private DnsQueryErrorHandler errorHandler; public HappyDns(){ - int dnsTimeout = GlobalConfiguration.getInstance().dnsServerResolveTimeout; + int dnsTimeout = GlobalConfiguration.getInstance().dnsResolveTimeout; systemDns = new SystemDns(dnsTimeout); - serversDns = new ServersDns(GlobalConfiguration.getInstance().dnsServers, dnsTimeout); httpDns = new HttpDns(dnsTimeout); } @@ -51,16 +41,6 @@ public List lookup(String hostname) throws UnknownHostExcept return addressList; } - // 指定 server ip dns - try { - addressList = serversDns.lookup(hostname); - } catch (IOException e) { - handleDnsError(e, hostname); - } - if (addressList != null && addressList.size() > 0) { - return addressList; - } - // http dns try { addressList = httpDns.lookup(hostname); diff --git a/library/src/main/java/com/qiniu/android/http/dns/ServersDns.java b/library/src/main/java/com/qiniu/android/http/dns/ServersDns.java deleted file mode 100644 index 8e55d4ca..00000000 --- a/library/src/main/java/com/qiniu/android/http/dns/ServersDns.java +++ /dev/null @@ -1,108 +0,0 @@ -package com.qiniu.android.http.dns; - -import com.qiniu.android.dns.Domain; -import com.qiniu.android.dns.IResolver; -import com.qiniu.android.dns.Record; -import com.qiniu.android.dns.local.Resolver; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.List; - -public class ServersDns implements Dns { - - private ArrayList customServers = new ArrayList<>(); - - public ServersDns(String[] dnsServers, int timeout) { - if (timeout < 0 || timeout > 20) { - timeout = 2; - } - - for (String dnsServer : dnsServers) { - try { - InetAddress address = InetAddress.getByName(dnsServer); - customServers.add(new CustomServer(address, timeout)); - } catch (Exception ignore) { - } - } - } - - @Override - public List lookup(String hostname) throws UnknownHostException { - if (customServers.size() == 0) { - return null; - } - - List servers = getCustomServers(); - - String serverIP = null; - Record[] records = null; - for (CustomServer server : servers) { - try { - records = server.resolver.resolve(new Domain(hostname), null); - } catch (IOException ignore) { - } - - if (records != null && records.length > 0) { - serverIP = server.serverIP; - server.addReliability(); - break; - } - } - - ArrayList addressList = new ArrayList<>(); - for (Record record : records) { - DnsNetworkAddress address = new DnsNetworkAddress(hostname, record.value, record.timeStamp, serverIP, (new Date()).getTime()); - addressList.add(address); - } - - return addressList; - } - - private List getCustomServers() { - List servers = (List) customServers.clone(); - Comparator comparator = new Comparator() { - @Override - public int compare(CustomServer o1, CustomServer o2) { - if (o1.isMoreReliabilityThan(o2)) { - return -1; - } else { - return 1; - } - } - }; - - Collections.sort(servers, comparator); - return servers; - } - - private static class CustomServer { - - private String serverIP = null; - private int reliability = 0; - private IResolver resolver; - - CustomServer(InetAddress address, int timeout) { - this.resolver = new Resolver(address, timeout); - this.serverIP = address.getHostName(); - } - - private void addReliability() { - reliability += 1; - } - - private boolean isMoreReliabilityThan(CustomServer customServer) { - return reliability > customServer.reliability; - } - - @Override - public String toString() { - return "{" + "\"" + "reliability" + "\":" + reliability + "}"; - } - } -} diff --git a/library/src/main/java/com/qiniu/android/http/metrics/UploadSingleRequestMetrics.java b/library/src/main/java/com/qiniu/android/http/metrics/UploadSingleRequestMetrics.java index 7c218946..51a79571 100644 --- a/library/src/main/java/com/qiniu/android/http/metrics/UploadSingleRequestMetrics.java +++ b/library/src/main/java/com/qiniu/android/http/metrics/UploadSingleRequestMetrics.java @@ -48,6 +48,8 @@ public class UploadSingleRequestMetrics extends UploadMetrics { public String remoteAddress; public Integer remotePort; + private long totalBytes = 0; + public long totalDnsTime(){ return time(domainLookupStartDate, domainLookupEndDate); } @@ -72,22 +74,21 @@ public long totalResponseTime(){ public void setRequest(Request request){ if (request != null){ this.request = new Request(request.urlString, request.httpMethod, request.allHeaders, null, request.timeout); + + long headerLength = 0; + long bodyLength = 0 ; + if (request.allHeaders != null){ + headerLength = (new JSONObject(request.allHeaders)).toString().length(); + } + if (request.httpBody != null){ + bodyLength = request.httpBody.length; + } + totalBytes = headerLength + bodyLength; } } public long totalBytes(){ - if (request == null){ - return 0; - } - long headerLength = 0; - long bodyLength = 0 ; - if (request.allHeaders != null){ - headerLength = (new JSONObject(request.allHeaders)).toString().length(); - } - if (request.httpBody != null){ - bodyLength = request.httpBody.length; - } - return (headerLength + bodyLength); + return totalBytes; } public Long bytesSend(){ diff --git a/library/src/main/java/com/qiniu/android/storage/BaseUpload.java b/library/src/main/java/com/qiniu/android/storage/BaseUpload.java index 8ebf4503..d73d619d 100644 --- a/library/src/main/java/com/qiniu/android/storage/BaseUpload.java +++ b/library/src/main/java/com/qiniu/android/storage/BaseUpload.java @@ -148,9 +148,6 @@ protected void completeAction(ResponseInfo responseInfo, JSONObject response) { if (completionHandler != null) { completionHandler.complete(responseInfo, key, metrics, response); } - - metrics = null; - currentRegionRequestMetrics = null; } private boolean setupRegions() { diff --git a/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java b/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java index 49c435e2..68693034 100644 --- a/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java +++ b/library/src/main/java/com/qiniu/android/storage/GlobalConfiguration.java @@ -59,15 +59,10 @@ public class GlobalConfiguration { public int partialHostFrozenTime = 5 * 60; /** - * 额外指定的 dns 预解析 server，当使用系统 dns 预解析失败时会使用 dnsServers 的顺序逐个进行预解析 - */ - public String[] dnsServers = new String[]{"114.114.114.114", "8.8.8.8", "119.29.29.29", "208.67.222.222", "223.5.5.5", "1.1.1.1"}; - - /** - * 额外指定的 {@link GlobalConfiguration#dnsServers} 在 dns 预取时的超时时间 + * 在 dns 预取时的超时时间 * 单位：秒 */ - public int dnsServerResolveTimeout = 2; + public int dnsResolveTimeout = 2; /** * 网络连接状态检测使用的connectCheckURLStrings，网络链接状态检测可能会影响重试机制，启动网络连接状态检测有助于提高上传可用性。 diff --git a/library/src/main/java/com/qiniu/android/storage/PartsUpload.java b/library/src/main/java/com/qiniu/android/storage/PartsUpload.java index ffea7bec..dfb7ad8e 100644 --- a/library/src/main/java/com/qiniu/android/storage/PartsUpload.java +++ b/library/src/main/java/com/qiniu/android/storage/PartsUpload.java @@ -246,12 +246,14 @@ public void complete(ResponseInfo responseInfo, UploadRegionRequestMetrics reque @Override protected void completeAction(ResponseInfo responseInfo, JSONObject response) { - reportBlock(); uploadPerformer.closeFile(); if (shouldRemoveUploadInfoRecord(responseInfo)) { uploadPerformer.removeUploadInfoRecord(); } + super.completeAction(responseInfo, response); + + reportBlock(); } private boolean shouldRemoveUploadInfoRecord(ResponseInfo responseInfo) { From 6162e32827e2d3daf40caab78daa18ae0968768f Mon Sep 17 00:00:00 2001 From: YangSen-qn Date: Mon, 9 Aug 2021 18:31:06 +0800 Subject: [PATCH 13/20] optimize http client metrics --- library/library.iml | 6 +++--- .../com/qiniu/android/http/request/HttpRegionRequest.java | 1 + .../android/http/request/httpclient/SystemHttpClient.java | 6 +++--- .../android/http/serverRegion/UploadDomainRegion.java | 8 +++++--- .../src/main/java/com/qiniu/android/utils/AsyncRun.java | 4 +++- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/library/library.iml b/library/library.iml index bbb68de1..dae8fc9d 100644 --- a/library/library.iml +++ b/library/library.iml @@ -4,7 +4,7 @@ @@ -19,8 +19,8 @@