From ded534973bd49e4f854904c79fc9b27d2135252f Mon Sep 17 00:00:00 2001 From: chzyer Date: Sun, 11 Aug 2013 20:12:04 +0800 Subject: [PATCH 01/16] fix multipart content length, print exception info --- src/com/qiniu/auth/Client.java | 8 +++++--- src/com/qiniu/auth/JSONObjectRet.java | 1 + src/com/qiniu/utils/MultipartEntity.java | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/com/qiniu/auth/Client.java b/src/com/qiniu/auth/Client.java index 330399e3a..12a1fdefc 100644 --- a/src/com/qiniu/auth/Client.java +++ b/src/com/qiniu/auth/Client.java @@ -78,9 +78,6 @@ protected Object doInBackground(Object... objects) { int statusCode = resp.getStatusLine().getStatusCode(); - if (statusCode / 100 != 2) { - return new Exception(errMsg); - } byte[] data = new byte[0]; try { @@ -89,6 +86,11 @@ protected Object doInBackground(Object... objects) { e.printStackTrace(); } + if (statusCode / 100 != 2) { + errMsg += new String(data); + return new Exception(errMsg); + } + return data; } diff --git a/src/com/qiniu/auth/JSONObjectRet.java b/src/com/qiniu/auth/JSONObjectRet.java index e511f13e7..46bc12873 100644 --- a/src/com/qiniu/auth/JSONObjectRet.java +++ b/src/com/qiniu/auth/JSONObjectRet.java @@ -13,6 +13,7 @@ public void onSuccess(byte[] body) { JSONObject obj = new JSONObject(new String(body)); onSuccess(obj); } catch (JSONException e) { + e.printStackTrace(); onFailure(new Exception(new String(body))); } } diff --git a/src/com/qiniu/utils/MultipartEntity.java b/src/com/qiniu/utils/MultipartEntity.java index bf2e06ae6..e9a0a3203 100644 --- a/src/com/qiniu/utils/MultipartEntity.java +++ b/src/com/qiniu/utils/MultipartEntity.java @@ -35,7 +35,7 @@ public boolean isRepeatable() { @Override public long getContentLength() { - long len = mData.length(); + long len = mData.toString().getBytes().length; for (FileInfo fi: mFiles) { len += fi.length(); } From 7bfa23dcdab86af23aed9450ce614d9675998a33 Mon Sep 17 00:00:00 2001 From: chzyer Date: Fri, 16 Aug 2013 09:47:31 +0800 Subject: [PATCH 02/16] update README: add download url --- docs/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/README.md b/docs/README.md index 008ef2b51..426547426 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,6 +2,9 @@ title: Android SDK 使用指南 --- +- Android SDK 下载地址： +- Android SDK 源码地址： (请注意非 master 分支的代码在规格上可能承受变更) + 此 Android SDK 基于 [七牛云存储官方API](http://docs.qiniu.com/api/index.html) 构建。在开发者的 Android App 工程项目中使用此 SDK 能够非常方便地将 Android 系统里边的文件快速直传到七牛云存储。 出于安全考虑，使用此 SDK 无需设置密钥（AccessKey / SecretKey）。所有涉及到授权的操作，比如生成上传授权凭证（uploadToken）或下载授权凭证（downloadToken）均在业务服务器端进行。 From 2a422aaa28ce210a28ffbbdc8bf7d286fe521c22 Mon Sep 17 00:00:00 2001 From: chzyer Date: Tue, 27 Aug 2013 17:05:10 +0800 Subject: [PATCH 03/16] fix client response --- src/com/qiniu/auth/Client.java | 32 +++++++------------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/src/com/qiniu/auth/Client.java b/src/com/qiniu/auth/Client.java index 12a1fdefc..18fd0b53f 100644 --- a/src/com/qiniu/auth/Client.java +++ b/src/com/qiniu/auth/Client.java @@ -62,36 +62,18 @@ class ClientExecuter extends AsyncTask { protected Object doInBackground(Object... objects) { httpPost = (HttpPost) objects[0]; ret = (CallRet) objects[1]; - String errMsg = ""; - - HttpResponse resp; try { - resp = roundtrip(httpPost); + HttpResponse resp = roundtrip(httpPost); + byte[] data = EntityUtils.toByteArray(resp.getEntity()); + int statusCode = resp.getStatusLine().getStatusCode(); + if (statusCode / 100 != 2) { + return new Exception(new String(data)); + } + return data; } catch (IOException e) { e.printStackTrace(); return e; } - - if (resp.getHeaders("X-Log").length > 0) { - errMsg = resp.getHeaders("X-Log")[0].getValue(); - } - - int statusCode = resp.getStatusLine().getStatusCode(); - - - byte[] data = new byte[0]; - try { - data = EntityUtils.toByteArray(resp.getEntity()); - } catch (IOException e) { - e.printStackTrace(); - } - - if (statusCode / 100 != 2) { - errMsg += new String(data); - return new Exception(errMsg); - } - - return data; } @Override From bf6ee90c813dd802ee26069c221ff9fc5fb2a1d9 Mon Sep 17 00:00:00 2001 From: Cheney Date: Sun, 1 Sep 2013 23:42:38 +0800 Subject: [PATCH 04/16] add onProcess --- src/com/qiniu/auth/CallRet.java | 6 ++-- src/com/qiniu/auth/Client.java | 21 +++++++++-- src/com/qiniu/io/IO.java | 14 +++++--- src/com/qiniu/utils/IOnProcess.java | 5 +++ src/com/qiniu/utils/InputStreamAt.java | 45 +++++++++++++++++++++++- src/com/qiniu/utils/MultipartEntity.java | 27 ++++++++++++-- 6 files changed, 106 insertions(+), 12 deletions(-) create mode 100644 src/com/qiniu/utils/IOnProcess.java diff --git a/src/com/qiniu/auth/CallRet.java b/src/com/qiniu/auth/CallRet.java index b2a061e0c..aac09905a 100644 --- a/src/com/qiniu/auth/CallRet.java +++ b/src/com/qiniu/auth/CallRet.java @@ -1,7 +1,9 @@ package com.qiniu.auth; -public abstract class CallRet { +import com.qiniu.utils.IOnProcess; + +public abstract class CallRet implements IOnProcess { public abstract void onSuccess(byte[] body); public abstract void onFailure(Exception ex); - public void onSuccess(){} + public void onProcess(long current, long total){} } \ No newline at end of file diff --git a/src/com/qiniu/auth/Client.java b/src/com/qiniu/auth/Client.java index 18fd0b53f..67c7d7ed2 100644 --- a/src/com/qiniu/auth/Client.java +++ b/src/com/qiniu/auth/Client.java @@ -2,6 +2,7 @@ import android.os.AsyncTask; import com.qiniu.conf.Conf; +import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; @@ -32,7 +33,12 @@ public void call(String url, CallRet ret) { } public void call(String url, HttpEntity entity, CallRet ret) { - call(url, entity.getContentType().getValue(), entity, ret); + Header header = entity.getContentType(); + String contentType = "application/octet-stream"; + if (header != null) { + contentType = header.getValue(); + } + call(url, contentType, entity, ret); } public void call(String url, String contentType, HttpEntity entity, CallRet ret) { @@ -64,9 +70,20 @@ protected Object doInBackground(Object... objects) { ret = (CallRet) objects[1]; try { HttpResponse resp = roundtrip(httpPost); + int statusCode = resp.getStatusLine().getStatusCode(); + if (statusCode == 401) { // android 2.3 will not response + return new Exception(resp.getStatusLine().getReasonPhrase()); + } byte[] data = EntityUtils.toByteArray(resp.getEntity()); - int statusCode = resp.getStatusLine().getStatusCode(); + if (statusCode / 100 != 2) { + if (data.length == 0) { + String xlog = resp.getFirstHeader("X-Log").getValue(); + if (xlog.length() > 0) { + return new Exception(xlog); + } + return new Exception(resp.getStatusLine().getReasonPhrase()); + } return new Exception(new String(data)); } return data; diff --git a/src/com/qiniu/io/IO.java b/src/com/qiniu/io/IO.java index a7e89b169..d07dd16ba 100644 --- a/src/com/qiniu/io/IO.java +++ b/src/com/qiniu/io/IO.java @@ -14,7 +14,7 @@ public class IO { - public static String UNDEFINED_KEY = "?"; + public static String UNDEFINED_KEY = null; private static Client mClient; @@ -49,7 +49,7 @@ public static void put(String uptoken, String key, InputStreamAt isa, PutExtra e } if (uptoken == null || uptoken.length() == 0) { - ret.onFailure(new Exception("uptoken未提供")); + ret.onFailure(new Exception("uptoken not specify")); return; } @@ -66,6 +66,7 @@ public static void put(String uptoken, String key, InputStreamAt isa, PutExtra e m.addField("token", uptoken); m.addFile("file", extra.mimeType, key, isa); + m.setProcessNotify(ret); defaultClient().call(Conf.UP_HOST, m, ret); @@ -97,8 +98,13 @@ public void onSuccess(JSONObject obj) { isa.close(); ret.onSuccess(obj); } - - @Override + + @Override + public void onProcess(long current, long total) { + ret.onProcess(current, total); + } + + @Override public void onFailure(Exception ex) { isa.close(); ret.onFailure(ex); diff --git a/src/com/qiniu/utils/IOnProcess.java b/src/com/qiniu/utils/IOnProcess.java new file mode 100644 index 000000000..718ee28db --- /dev/null +++ b/src/com/qiniu/utils/IOnProcess.java @@ -0,0 +1,5 @@ +package com.qiniu.utils; + +public interface IOnProcess { + public void onProcess(long current, long total); +} diff --git a/src/com/qiniu/utils/InputStreamAt.java b/src/com/qiniu/utils/InputStreamAt.java index 36c5d5fad..e9621e906 100644 --- a/src/com/qiniu/utils/InputStreamAt.java +++ b/src/com/qiniu/utils/InputStreamAt.java @@ -1,12 +1,18 @@ package com.qiniu.utils; import android.content.Context; +import org.apache.http.Header; +import org.apache.http.HeaderElement; +import org.apache.http.HttpEntity; +import org.apache.http.ParseException; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.entity.AbstractHttpEntity; import java.io.*; import java.util.Arrays; import java.util.zip.CRC32; -public class InputStreamAt implements Closeable{ +public class InputStreamAt implements Closeable { private RandomAccessFile mFileStream; private byte[] mData; @@ -167,4 +173,41 @@ public int read(byte[] data) throws IOException { return mFileStream.read(data); } + public HttpEntity toHttpEntity(final long offset, final int length) { + final InputStreamAt input = this; + return new AbstractHttpEntity() { + @Override + public boolean isRepeatable() { + return false; + } + + @Override + public long getContentLength() { + return length; + } + + @Override + public InputStream getContent() throws IOException, IllegalStateException { + return null; + } + + @Override + public void writeTo(OutputStream outputStream) throws IOException { + int blockSize = 128 * 1024; + long start = offset; + long end = offset + length; + while (start < end) { + int readLength = (int) StrictMath.min((long) blockSize, end-start); + outputStream.write(input.read(start, readLength)); + outputStream.flush(); + start += readLength; + } + } + + @Override + public boolean isStreaming() { + return false; + } + }; + } } diff --git a/src/com/qiniu/utils/MultipartEntity.java b/src/com/qiniu/utils/MultipartEntity.java index e9a0a3203..2dcfbad56 100644 --- a/src/com/qiniu/utils/MultipartEntity.java +++ b/src/com/qiniu/utils/MultipartEntity.java @@ -11,6 +11,9 @@ public class MultipartEntity extends AbstractHttpEntity { private String mBoundary; + private long mContentLength = -1; + private long writed = 0; + private IOnProcess mNotify; private StringBuffer mData = new StringBuffer(); private ArrayList mFiles = new ArrayList(); @@ -35,11 +38,13 @@ public boolean isRepeatable() { @Override public long getContentLength() { + if (mContentLength > 0) return mContentLength; long len = mData.toString().getBytes().length; for (FileInfo fi: mFiles) { len += fi.length(); } len += 6 + mBoundary.length(); + mContentLength = len; return len; } @@ -52,11 +57,16 @@ public InputStream getContent() throws IOException, IllegalStateException { public void writeTo(OutputStream outputStream) throws IOException { outputStream.write(mData.toString().getBytes()); outputStream.flush(); + writed += mData.toString().getBytes().length; + if (mNotify != null) mNotify.onProcess(writed, getContentLength()); for (FileInfo i: mFiles) { i.writeTo(outputStream); } - outputStream.write(("--" + mBoundary + "--\r\n").getBytes()); + byte[] data = ("--" + mBoundary + "--\r\n").getBytes(); + outputStream.write(data); outputStream.flush(); + writed += data.length; + if (mNotify != null) mNotify.onProcess(writed, getContentLength()); outputStream.close(); } @@ -67,7 +77,11 @@ public boolean isStreaming() { private String fileTpl = "--%s\r\nContent-Disposition: form-data;name=\"%s\";filename=\"%s\"\r\nContent-Type: %s\r\n\r\n"; - class FileInfo { + public void setProcessNotify(IOnProcess ret) { + mNotify = ret; + } + + class FileInfo { public String mField; public String mContentType; @@ -90,8 +104,11 @@ public long length() { } public void writeTo(OutputStream outputStream) throws IOException { - outputStream.write(String.format(fileTpl, mBoundary, mField, mFilename, mContentType).getBytes()); + byte[] data = String.format(fileTpl, mBoundary, mField, mFilename, mContentType).getBytes(); + outputStream.write(data); outputStream.flush(); + writed += data.length; + if (mNotify != null) mNotify.onProcess(writed, getContentLength()); int blockSize = 256 * 1024; long index = 0; @@ -101,9 +118,13 @@ public void writeTo(OutputStream outputStream) throws IOException { outputStream.write(mIsa.read(index, readLength)); index += blockSize; outputStream.flush(); + writed += readLength; + if (mNotify != null) mNotify.onProcess(writed, getContentLength()); } outputStream.write("\r\n".getBytes()); outputStream.flush(); + writed += 2; + if (mNotify != null) mNotify.onProcess(writed, getContentLength()); } } From 6edc290954a2ecaf57f77fd7c841d1de931be742 Mon Sep 17 00:00:00 2001 From: Cheney Date: Sun, 1 Sep 2013 23:42:46 +0800 Subject: [PATCH 05/16] add resumable io --- src/com/qiniu/auth/UpClient.java | 32 ----- src/com/qiniu/resumableio/PutExtra.java | 9 ++ src/com/qiniu/resumableio/PutRet.java | 21 ++++ .../qiniu/resumableio/ResumableClient.java | 102 +++++++++++++++ src/com/qiniu/resumableio/ResumableIO.java | 118 ++++++++++++++++++ 5 files changed, 250 insertions(+), 32 deletions(-) delete mode 100644 src/com/qiniu/auth/UpClient.java create mode 100644 src/com/qiniu/resumableio/PutExtra.java create mode 100644 src/com/qiniu/resumableio/PutRet.java create mode 100644 src/com/qiniu/resumableio/ResumableClient.java create mode 100644 src/com/qiniu/resumableio/ResumableIO.java diff --git a/src/com/qiniu/auth/UpClient.java b/src/com/qiniu/auth/UpClient.java deleted file mode 100644 index 7105d283d..000000000 --- a/src/com/qiniu/auth/UpClient.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.qiniu.auth; - -import org.apache.http.HttpResponse; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpPost; - -import java.io.IOException; - -public class UpClient extends Client { - private String mUpToken; - - public UpClient(HttpClient client) { - super(client); - } - - public void updateToken(String token) { - mUpToken = token; - } - - @Override - protected HttpResponse roundtrip(HttpPost httpPost) throws IOException { - if (mUpToken != null) { - httpPost.setHeader("Authorization", "UpToken " + mUpToken); - } - return super.roundtrip(httpPost); - } - - public static UpClient defaultClient() { - return new UpClient(getMultithreadClient()); - } - -} diff --git a/src/com/qiniu/resumableio/PutExtra.java b/src/com/qiniu/resumableio/PutExtra.java new file mode 100644 index 000000000..2722bf4df --- /dev/null +++ b/src/com/qiniu/resumableio/PutExtra.java @@ -0,0 +1,9 @@ +package com.qiniu.resumableio; + +import java.util.Map; + +public class PutExtra { + public Map params; + public PutRet[] processes; + public String mimeType; +} diff --git a/src/com/qiniu/resumableio/PutRet.java b/src/com/qiniu/resumableio/PutRet.java new file mode 100644 index 000000000..53ad296eb --- /dev/null +++ b/src/com/qiniu/resumableio/PutRet.java @@ -0,0 +1,21 @@ +package com.qiniu.resumableio; + +import org.json.JSONObject; + +public class PutRet { + public String ctx; + public String host; + public int crc32; + public String checksum; + public int offset = 0; + public boolean isInvalid() { + return ctx == null || offset == 0; + } + public void parse(JSONObject obj) { + ctx = obj.optString("ctx", ""); + host = obj.optString("host", ""); + crc32 = obj.optInt("crc32", 0); + checksum = obj.optString("checksum", ""); + offset = obj.optInt("offset", 0); + } +} diff --git a/src/com/qiniu/resumableio/ResumableClient.java b/src/com/qiniu/resumableio/ResumableClient.java new file mode 100644 index 000000000..2de2af3b0 --- /dev/null +++ b/src/com/qiniu/resumableio/ResumableClient.java @@ -0,0 +1,102 @@ +package com.qiniu.resumableio; + +import android.util.Base64; +import com.qiniu.auth.CallRet; +import com.qiniu.auth.Client; +import com.qiniu.auth.JSONObjectRet; +import com.qiniu.conf.Conf; +import com.qiniu.utils.InputStreamAt; +import com.qiniu.utils.Slice; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.json.JSONObject; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.Map; + +public class ResumableClient extends Client { + String mUpToken; + int CHUNK_SIZE = 256 * 1024; + int BLOCK_SIZE = 4 * 1024 * 1024; + public ResumableClient(HttpClient client, String uptoken) { + super(client); + mUpToken = uptoken; + } + + @Override + protected HttpResponse roundtrip(HttpPost httpPost) throws IOException { + if (mUpToken != null) { + httpPost.setHeader("Authorization", "UpToken " + mUpToken); + } + return super.roundtrip(httpPost); + } + + public void putblock(final InputStreamAt input, final PutRet putRet, long offset, final JSONObjectRet callback) { + final long writeNeed = Math.min(input.length()-offset, BLOCK_SIZE); + JSONObjectRet ret = new JSONObjectRet() { + @Override + public void onSuccess(JSONObject obj) { + putRet.parse(obj); + if (putRet.offset == writeNeed) { + callback.onSuccess(obj); + return; + } + bput(putRet.host, input, putRet.ctx, putRet.offset, this); + } + + @Override + public void onFailure(Exception ex) { + callback.onFailure(ex); + } + }; + if (putRet.isInvalid()) { + mkblk(input, offset, ret); + } else { + bput(putRet.host, input, putRet.ctx, putRet.offset, ret); + } + } + + public void mkblk(InputStreamAt input, long offset, CallRet ret) { + int remainLength = Math.min((int) (input.length() - offset), BLOCK_SIZE); + String url = Conf.UP_HOST + "/mkblk/" + remainLength; + remainLength = Math.min((int) (input.length()-offset), CHUNK_SIZE); + call(url, input.toHttpEntity(offset, remainLength), ret); + } + + public void bput(String host, InputStreamAt input, String ctx, long offset, CallRet ret) { + int remainLength = Math.min((int) (input.length() - offset), CHUNK_SIZE); + String url = host + "/bput/" + ctx + "/" + offset; + call(url, input.toHttpEntity(offset, remainLength), ret); + } + + public void mkfile(String key, long fsize, String mimeType, Map params, String ctxs, CallRet ret) { + String url = Conf.UP_HOST + "/mkfile/" + fsize; + if (mimeType != null) { + url += "/mimeType/" + encode(mimeType); + } + if (key != null) { + url += "/key/" + encode(key); + } + if (params != null && params.size() > 0) { + for (Map.Entry a: params.entrySet()) { + url += "/" + a.getKey() + "/" + encode(a.getValue()); + } + } + StringEntity se; + try { + se = new StringEntity(ctxs); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + ret.onFailure(e); + return; + } + call(url, se, ret); + } + + public String encode(String data) { + return Base64.encodeToString(data.getBytes(), Base64.URL_SAFE | Base64.NO_WRAP); + } +} diff --git a/src/com/qiniu/resumableio/ResumableIO.java b/src/com/qiniu/resumableio/ResumableIO.java new file mode 100644 index 000000000..fb849819c --- /dev/null +++ b/src/com/qiniu/resumableio/ResumableIO.java @@ -0,0 +1,118 @@ +package com.qiniu.resumableio; + +import android.content.Context; +import android.net.Uri; +import com.qiniu.auth.CallRet; +import com.qiniu.auth.Client; +import com.qiniu.auth.JSONObjectRet; +import com.qiniu.demo.MyActivity; +import com.qiniu.utils.InputStreamAt; +import com.qiniu.utils.Slice; +import org.json.JSONObject; + +import java.io.FileNotFoundException; + +public class ResumableIO { + String mUptoken = ""; + ResumableClient mClient; + private int BLOCK_SIZE = 4 * 1024 * 1024; + private int CHUNK_SIZE = 256 * 1024; + public ResumableIO(String uptoken) { + mUptoken = uptoken; + } + public ResumableIO(ResumableClient client, String uptoken) { + mClient = client; + mUptoken = uptoken; + } + + public int put(final String key, final InputStreamAt input, final PutExtra extra, final JSONObjectRet ret) { + final int blkCount = (int) (input.length() / BLOCK_SIZE) + 1; + if (extra.processes == null) { + extra.processes = new PutRet[blkCount]; + for (int i=0; i 0) { + ctxes = ctxes.substring(1); + } + mClient.mkfile(key, input.length(), extra.mimeType, extra.params, ctxes, ret); + } + } + + @Override + public void onFailure(Exception ex) { + if (ex.getMessage().contains("Unauthorization")) { + ret.onFailure(ex); + return; + } + retryTime--; + if (retryTime <= 0) { + ret.onFailure(ex); + return; + } + mClient.putblock(input, process, startPos, this); + } + }); + } + return 0; + } + + public int putFile(Context mContext, String key, Uri uri, PutExtra extra, final JSONObjectRet ret) { + + final InputStreamAt isa; + try { + isa = InputStreamAt.fromInputStream(mContext, mContext.getContentResolver().openInputStream(uri)); + } catch (FileNotFoundException e) { + ret.onFailure(e); + return -1; + } + + put(key, isa, extra, new JSONObjectRet() { + @Override + public void onSuccess(JSONObject obj) { + isa.close(); + ret.onSuccess(obj); + } + + @Override + public void onProcess(long current, long total) { + ret.onProcess(current, total); + } + + @Override + public void onFailure(Exception ex) { + isa.close(); + ret.onFailure(ex); + } + }); + return -1; + } + + public static void Stop(int id) { + + } + + public static int put(String uptoken, String key, InputStreamAt isa, PutExtra extra, JSONObjectRet ret) { + return new ResumableIO(new ResumableClient(Client.getMultithreadClient(), uptoken), uptoken).put(key, isa, extra, ret); + } + + public static int putFile(Context mContext, String uptoken, String key, Uri uri, PutExtra extra, JSONObjectRet ret) { + return new ResumableIO(new ResumableClient(Client.getMultithreadClient(), uptoken), uptoken).putFile(mContext, key, uri, extra, ret); + } + +} From 199ffc57148996a99e23f96b5a5cefa3452e6b72 Mon Sep 17 00:00:00 2001 From: Cheney Date: Tue, 3 Sep 2013 00:29:14 +0800 Subject: [PATCH 06/16] add upload cancelable --- src/com/qiniu/auth/CallRet.java | 1 + src/com/qiniu/auth/Client.java | 21 ++++---- .../qiniu/resumableio/ResumableClient.java | 31 ++++++----- src/com/qiniu/resumableio/ResumableIO.java | 51 ++++++++++++------- src/com/qiniu/utils/ICancel.java | 5 ++ src/com/qiniu/utils/InputStreamAt.java | 6 ++- 6 files changed, 76 insertions(+), 39 deletions(-) create mode 100644 src/com/qiniu/utils/ICancel.java diff --git a/src/com/qiniu/auth/CallRet.java b/src/com/qiniu/auth/CallRet.java index aac09905a..8072ad653 100644 --- a/src/com/qiniu/auth/CallRet.java +++ b/src/com/qiniu/auth/CallRet.java @@ -6,4 +6,5 @@ public abstract class CallRet implements IOnProcess { public abstract void onSuccess(byte[] body); public abstract void onFailure(Exception ex); public void onProcess(long current, long total){} + public void onPause(){} } \ No newline at end of file diff --git a/src/com/qiniu/auth/Client.java b/src/com/qiniu/auth/Client.java index 67c7d7ed2..cdfb3df90 100644 --- a/src/com/qiniu/auth/Client.java +++ b/src/com/qiniu/auth/Client.java @@ -2,6 +2,7 @@ import android.os.AsyncTask; import com.qiniu.conf.Conf; +import com.qiniu.utils.ICancel; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -27,32 +28,34 @@ public Client(HttpClient client) { mClient = client; } - public void call(String url, CallRet ret) { + public ICancel call(String url, CallRet ret) { HttpPost httppost = new HttpPost(url); - execute(httppost, ret); + return execute(httppost, ret); } - public void call(String url, HttpEntity entity, CallRet ret) { + public ICancel call(String url, HttpEntity entity, CallRet ret) { Header header = entity.getContentType(); String contentType = "application/octet-stream"; if (header != null) { contentType = header.getValue(); } - call(url, contentType, entity, ret); + return call(url, contentType, entity, ret); } - public void call(String url, String contentType, HttpEntity entity, CallRet ret) { + public ICancel call(String url, String contentType, HttpEntity entity, CallRet ret) { HttpPost httppost = new HttpPost(url); httppost.setEntity(entity); if (contentType != null) { httppost.setHeader("Content-Type", contentType); } - execute(httppost, ret); + return execute(httppost, ret); } - protected void execute(HttpPost httpPost, CallRet ret) { - new ClientExecuter().execute(httpPost, ret); + protected ClientExecuter execute(HttpPost httpPost, CallRet ret) { + ClientExecuter client = new ClientExecuter(); + client.execute(httpPost, ret); + return client; } protected HttpResponse roundtrip(HttpPost httpPost) throws IOException { @@ -60,7 +63,7 @@ protected HttpResponse roundtrip(HttpPost httpPost) throws IOException { return mClient.execute(httpPost); } - class ClientExecuter extends AsyncTask { + class ClientExecuter extends AsyncTask implements ICancel { HttpPost httpPost; CallRet ret; diff --git a/src/com/qiniu/resumableio/ResumableClient.java b/src/com/qiniu/resumableio/ResumableClient.java index 2de2af3b0..b6a7da63d 100644 --- a/src/com/qiniu/resumableio/ResumableClient.java +++ b/src/com/qiniu/resumableio/ResumableClient.java @@ -5,8 +5,8 @@ import com.qiniu.auth.Client; import com.qiniu.auth.JSONObjectRet; import com.qiniu.conf.Conf; +import com.qiniu.utils.ICancel; import com.qiniu.utils.InputStreamAt; -import com.qiniu.utils.Slice; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpPost; @@ -34,8 +34,9 @@ protected HttpResponse roundtrip(HttpPost httpPost) throws IOException { return super.roundtrip(httpPost); } - public void putblock(final InputStreamAt input, final PutRet putRet, long offset, final JSONObjectRet callback) { + public ICancel[] putblock(final InputStreamAt input, final PutRet putRet, long offset, final JSONObjectRet callback) { final long writeNeed = Math.min(input.length()-offset, BLOCK_SIZE); + final ICancel[] canceler = new ICancel[] {null}; JSONObjectRet ret = new JSONObjectRet() { @Override public void onSuccess(JSONObject obj) { @@ -44,7 +45,12 @@ public void onSuccess(JSONObject obj) { callback.onSuccess(obj); return; } - bput(putRet.host, input, putRet.ctx, putRet.offset, this); + canceler[0] = bput(putRet.host, input, putRet.ctx, putRet.offset, this); + } + + @Override + public void onProcess(long current, long total) { + callback.onProcess(current, writeNeed); } @Override @@ -53,26 +59,27 @@ public void onFailure(Exception ex) { } }; if (putRet.isInvalid()) { - mkblk(input, offset, ret); + canceler[0] = mkblk(input, offset, ret); } else { - bput(putRet.host, input, putRet.ctx, putRet.offset, ret); + canceler[0] = bput(putRet.host, input, putRet.ctx, putRet.offset, ret); } + return canceler; } - public void mkblk(InputStreamAt input, long offset, CallRet ret) { + public ICancel mkblk(InputStreamAt input, long offset, CallRet ret) { int remainLength = Math.min((int) (input.length() - offset), BLOCK_SIZE); String url = Conf.UP_HOST + "/mkblk/" + remainLength; remainLength = Math.min((int) (input.length()-offset), CHUNK_SIZE); - call(url, input.toHttpEntity(offset, remainLength), ret); + return call(url, input.toHttpEntity(offset, remainLength, ret), ret); } - public void bput(String host, InputStreamAt input, String ctx, long offset, CallRet ret) { + public ICancel bput(String host, InputStreamAt input, String ctx, long offset, CallRet ret) { int remainLength = Math.min((int) (input.length() - offset), CHUNK_SIZE); String url = host + "/bput/" + ctx + "/" + offset; - call(url, input.toHttpEntity(offset, remainLength), ret); + return call(url, input.toHttpEntity(offset, remainLength, ret), ret); } - public void mkfile(String key, long fsize, String mimeType, Map params, String ctxs, CallRet ret) { + public ICancel mkfile(String key, long fsize, String mimeType, Map params, String ctxs, CallRet ret) { String url = Conf.UP_HOST + "/mkfile/" + fsize; if (mimeType != null) { url += "/mimeType/" + encode(mimeType); @@ -91,9 +98,9 @@ public void mkfile(String key, long fsize, String mimeType, Map } catch (UnsupportedEncodingException e) { e.printStackTrace(); ret.onFailure(e); - return; + return null; } - call(url, se, ret); + return call(url, se, ret); } public String encode(String data) { diff --git a/src/com/qiniu/resumableio/ResumableIO.java b/src/com/qiniu/resumableio/ResumableIO.java index fb849819c..71a1550c5 100644 --- a/src/com/qiniu/resumableio/ResumableIO.java +++ b/src/com/qiniu/resumableio/ResumableIO.java @@ -2,27 +2,32 @@ import android.content.Context; import android.net.Uri; -import com.qiniu.auth.CallRet; import com.qiniu.auth.Client; import com.qiniu.auth.JSONObjectRet; -import com.qiniu.demo.MyActivity; +import com.qiniu.utils.ICancel; import com.qiniu.utils.InputStreamAt; -import com.qiniu.utils.Slice; import org.json.JSONObject; import java.io.FileNotFoundException; +import java.util.HashMap; public class ResumableIO { - String mUptoken = ""; ResumableClient mClient; private int BLOCK_SIZE = 4 * 1024 * 1024; private int CHUNK_SIZE = 256 * 1024; + private static int atomicId = 0; + private static HashMap idCancels = new HashMap(); + public ResumableIO(String uptoken) { - mUptoken = uptoken; + mClient = new ResumableClient(Client.getMultithreadClient(), uptoken); } - public ResumableIO(ResumableClient client, String uptoken) { + public ResumableIO(ResumableClient client) { mClient = client; - mUptoken = uptoken; + } + + private synchronized Integer newTask(ICancel[][] c) { + idCancels.put(atomicId, c); + return atomicId++; } public int put(final String key, final InputStreamAt input, final PutExtra extra, final JSONObjectRet ret) { @@ -34,10 +39,13 @@ public int put(final String key, final InputStreamAt input, final PutExtra extra } } final int[] success = new int[] {0}; + final long[] uploaded = new long[blkCount]; + final ICancel[][] cancelers = new ICancel[blkCount][1]; for (int i=0; i Date: Wed, 4 Sep 2013 15:11:02 +0800 Subject: [PATCH 07/16] update utile --- src/com/qiniu/utils/ICancel.java | 2 +- src/com/qiniu/utils/IOnProcess.java | 2 +- src/com/qiniu/utils/InputStreamAt.java | 159 +++++++++++++---------- src/com/qiniu/utils/MultipartEntity.java | 42 +++--- 4 files changed, 116 insertions(+), 89 deletions(-) diff --git a/src/com/qiniu/utils/ICancel.java b/src/com/qiniu/utils/ICancel.java index 75626fb79..96e979ae2 100644 --- a/src/com/qiniu/utils/ICancel.java +++ b/src/com/qiniu/utils/ICancel.java @@ -1,5 +1,5 @@ package com.qiniu.utils; public interface ICancel { - public boolean cancel(boolean isIntercupt); + public boolean cancel(boolean isIntercupt); } diff --git a/src/com/qiniu/utils/IOnProcess.java b/src/com/qiniu/utils/IOnProcess.java index 718ee28db..9d45a08f2 100644 --- a/src/com/qiniu/utils/IOnProcess.java +++ b/src/com/qiniu/utils/IOnProcess.java @@ -1,5 +1,5 @@ package com.qiniu.utils; public interface IOnProcess { - public void onProcess(long current, long total); + public void onProcess(long current, long total); } diff --git a/src/com/qiniu/utils/InputStreamAt.java b/src/com/qiniu/utils/InputStreamAt.java index d20dbe8b2..639f832e2 100644 --- a/src/com/qiniu/utils/InputStreamAt.java +++ b/src/com/qiniu/utils/InputStreamAt.java @@ -1,6 +1,8 @@ package com.qiniu.utils; import android.content.Context; +import android.os.Environment; +import com.qiniu.auth.Client; import org.apache.http.Header; import org.apache.http.HeaderElement; import org.apache.http.HttpEntity; @@ -18,7 +20,9 @@ public class InputStreamAt implements Closeable { private File mTmpFile; private boolean mClosed; + private boolean mDelWhenClose = false; private long mCrc32 = -1; + private long mLength = -1; /** * @param context @@ -26,7 +30,13 @@ public class InputStreamAt implements Closeable { */ public static InputStreamAt fromInputStream(Context context, InputStream is) { File file = storeToFile(context, is); - InputStreamAt isa = new InputStreamAt(file); + if (file == null) return null; + InputStreamAt isa = new InputStreamAt(file, true); + return isa; + } + + public static InputStreamAt fromFile(File f) { + InputStreamAt isa = new InputStreamAt(f); return isa; } @@ -35,7 +45,12 @@ public static InputStreamAt fromString(String str) { } public InputStreamAt(File file) { + this(file, false); + } + + public InputStreamAt(File file, boolean delWhenClose) { mTmpFile = file; + mDelWhenClose = delWhenClose; try { mFileStream = new RandomAccessFile(mTmpFile, "r"); } catch (FileNotFoundException e) { @@ -47,6 +62,13 @@ public InputStreamAt(byte[] data) { mData = data; } + public long getCrc32(long offset, int length) { + CRC32 crc32 = new CRC32(); + byte[] data = read(offset, length); + crc32.update(data); + return crc32.getValue(); + } + public long crc32() { if (mCrc32 >= 0) return mCrc32; CRC32 crc32 = new CRC32(); @@ -66,13 +88,16 @@ public long crc32() { } public long length() { + if (mLength >= 0) return mLength; if (mData != null) { - return mData.length; + mLength = mData.length; + return mLength; } if (mFileStream != null) { try { - return mFileStream.length(); + mLength = mFileStream.length(); + return mLength; } catch (IOException e) { e.printStackTrace(); } @@ -81,35 +106,22 @@ public long length() { } protected static File storeToFile(Context context, InputStream is) { - File outputDir = context.getCacheDir(); // context being the Activity pointer - File f = null; - OutputStream os = null; try { - f = File.createTempFile("qiniu-", "", outputDir); - os = new FileOutputStream(f); - byte[] buffer = new byte[4096]; + File outputDir = getSDPath(context); + File f = File.createTempFile("qiniu-", "", outputDir); + OutputStream os = new FileOutputStream(f); + byte[] buffer = new byte[64 * 1024]; int bytesRead; while ((bytesRead = is.read(buffer)) != -1) { os.write(buffer, 0, bytesRead); } + if (os != null) os.close(); + if (is != null) is.close(); + return f; } catch (IOException e) { e.printStackTrace(); + return null; } - if (os != null) { - try { - os.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - if (is != null) { - try { - is.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - return f; } public byte[] read(long offset, int length) { @@ -164,7 +176,7 @@ public synchronized void close(){ } } - if (mTmpFile != null) { + if (mTmpFile != null && mDelWhenClose) { mTmpFile.delete(); } } @@ -173,45 +185,60 @@ public int read(byte[] data) throws IOException { return mFileStream.read(data); } - public HttpEntity toHttpEntity(final long offset, final int length, final IOnProcess onProcess) { - final InputStreamAt input = this; - return new AbstractHttpEntity() { - @Override - public boolean isRepeatable() { - return false; - } - - @Override - public long getContentLength() { - return length; - } - - @Override - public InputStream getContent() throws IOException, IllegalStateException { - return null; - } - - @Override - public void writeTo(OutputStream outputStream) throws IOException { - int blockSize = 128 * 1024; - long start = offset; - long initStart = start; - long end = offset + length; - long total = end - start; - while (start < end) { - int readLength = (int) StrictMath.min((long) blockSize, end-start); - outputStream.write(input.read(start, readLength)); - outputStream.flush(); - initStart += readLength; - onProcess.onProcess(initStart, total); - start += readLength; - } - } - - @Override - public boolean isStreaming() { - return false; - } - }; - } + public HttpEntity toHttpEntity(final long offset, final int length, final Client.ClientExecuter client) { + final InputStreamAt input = this; + return new AbstractHttpEntity() { + @Override + public boolean isRepeatable() { + return false; + } + + @Override + public long getContentLength() { + return length; + } + + @Override + public InputStream getContent() throws IOException, IllegalStateException { + return null; + } + + @Override + public void writeTo(OutputStream outputStream) throws IOException { + int blockSize = 64 * 1024; + long start = offset; + long initStart = 0; + long end = offset + length; + long total = end - start; + while (start < end) { + if (mClosed) { + outputStream.close(); + return; + } + int readLength = (int) StrictMath.min((long) blockSize, end-start); + byte[] data = input.read(start, readLength); + outputStream.write(data); + outputStream.flush(); + initStart += readLength; + client.upload(initStart, total); + start += readLength; + } + } + + @Override + public boolean isStreaming() { + return false; + } + }; + } + + public static File getSDPath(Context context){ + File sdDir = context.getCacheDir(); + boolean sdCardExist = Environment.getExternalStorageState() + .equals(android.os.Environment.MEDIA_MOUNTED); + if (sdCardExist) { + sdDir = Environment.getExternalStorageDirectory(); + } + return sdDir; + } } diff --git a/src/com/qiniu/utils/MultipartEntity.java b/src/com/qiniu/utils/MultipartEntity.java index 2dcfbad56..c4a7b6d80 100644 --- a/src/com/qiniu/utils/MultipartEntity.java +++ b/src/com/qiniu/utils/MultipartEntity.java @@ -11,9 +11,9 @@ public class MultipartEntity extends AbstractHttpEntity { private String mBoundary; - private long mContentLength = -1; - private long writed = 0; - private IOnProcess mNotify; + private long mContentLength = -1; + private long writed = 0; + private IOnProcess mNotify; private StringBuffer mData = new StringBuffer(); private ArrayList mFiles = new ArrayList(); @@ -38,13 +38,13 @@ public boolean isRepeatable() { @Override public long getContentLength() { - if (mContentLength > 0) return mContentLength; + if (mContentLength > 0) return mContentLength; long len = mData.toString().getBytes().length; for (FileInfo fi: mFiles) { len += fi.length(); } len += 6 + mBoundary.length(); - mContentLength = len; + mContentLength = len; return len; } @@ -57,16 +57,16 @@ public InputStream getContent() throws IOException, IllegalStateException { public void writeTo(OutputStream outputStream) throws IOException { outputStream.write(mData.toString().getBytes()); outputStream.flush(); - writed += mData.toString().getBytes().length; - if (mNotify != null) mNotify.onProcess(writed, getContentLength()); + writed += mData.toString().getBytes().length; + if (mNotify != null) mNotify.onProcess(writed, getContentLength()); for (FileInfo i: mFiles) { i.writeTo(outputStream); } - byte[] data = ("--" + mBoundary + "--\r\n").getBytes(); + byte[] data = ("--" + mBoundary + "--\r\n").getBytes(); outputStream.write(data); outputStream.flush(); - writed += data.length; - if (mNotify != null) mNotify.onProcess(writed, getContentLength()); + writed += data.length; + if (mNotify != null) mNotify.onProcess(writed, getContentLength()); outputStream.close(); } @@ -77,11 +77,11 @@ public boolean isStreaming() { private String fileTpl = "--%s\r\nContent-Disposition: form-data;name=\"%s\";filename=\"%s\"\r\nContent-Type: %s\r\n\r\n"; - public void setProcessNotify(IOnProcess ret) { - mNotify = ret; - } + public void setProcessNotify(IOnProcess ret) { + mNotify = ret; + } - class FileInfo { + class FileInfo { public String mField; public String mContentType; @@ -104,11 +104,11 @@ public long length() { } public void writeTo(OutputStream outputStream) throws IOException { - byte[] data = String.format(fileTpl, mBoundary, mField, mFilename, mContentType).getBytes(); + byte[] data = String.format(fileTpl, mBoundary, mField, mFilename, mContentType).getBytes(); outputStream.write(data); outputStream.flush(); - writed += data.length; - if (mNotify != null) mNotify.onProcess(writed, getContentLength()); + writed += data.length; + if (mNotify != null) mNotify.onProcess(writed, getContentLength()); int blockSize = 256 * 1024; long index = 0; @@ -118,13 +118,13 @@ public void writeTo(OutputStream outputStream) throws IOException { outputStream.write(mIsa.read(index, readLength)); index += blockSize; outputStream.flush(); - writed += readLength; - if (mNotify != null) mNotify.onProcess(writed, getContentLength()); + writed += readLength; + if (mNotify != null) mNotify.onProcess(writed, getContentLength()); } outputStream.write("\r\n".getBytes()); outputStream.flush(); - writed += 2; - if (mNotify != null) mNotify.onProcess(writed, getContentLength()); + writed += 2; + if (mNotify != null) mNotify.onProcess(writed, getContentLength()); } } From d0997b40c302b04fa02725814f2e9d3b8d052f48 Mon Sep 17 00:00:00 2001 From: Cheney Date: Wed, 4 Sep 2013 15:11:22 +0800 Subject: [PATCH 08/16] update resumable io --- src/com/qiniu/resumableio/PutExtra.java | 6 +- src/com/qiniu/resumableio/PutRet.java | 30 +- .../qiniu/resumableio/ResumableClient.java | 166 ++++++---- src/com/qiniu/resumableio/ResumableIO.java | 309 +++++++++++------- 4 files changed, 303 insertions(+), 208 deletions(-) diff --git a/src/com/qiniu/resumableio/PutExtra.java b/src/com/qiniu/resumableio/PutExtra.java index 2722bf4df..0a3b9e44a 100644 --- a/src/com/qiniu/resumableio/PutExtra.java +++ b/src/com/qiniu/resumableio/PutExtra.java @@ -3,7 +3,7 @@ import java.util.Map; public class PutExtra { - public Map params; - public PutRet[] processes; - public String mimeType; + public Map params; + public PutRet[] processes; + public String mimeType; } diff --git a/src/com/qiniu/resumableio/PutRet.java b/src/com/qiniu/resumableio/PutRet.java index 53ad296eb..0217e36c1 100644 --- a/src/com/qiniu/resumableio/PutRet.java +++ b/src/com/qiniu/resumableio/PutRet.java @@ -3,19 +3,19 @@ import org.json.JSONObject; public class PutRet { - public String ctx; - public String host; - public int crc32; - public String checksum; - public int offset = 0; - public boolean isInvalid() { - return ctx == null || offset == 0; - } - public void parse(JSONObject obj) { - ctx = obj.optString("ctx", ""); - host = obj.optString("host", ""); - crc32 = obj.optInt("crc32", 0); - checksum = obj.optString("checksum", ""); - offset = obj.optInt("offset", 0); - } + public String ctx; + public String host; + public long crc32; + public String checksum; + public int offset = 0; + public boolean isInvalid() { + return ctx == null || offset == 0; + } + public void parse(JSONObject obj) { + ctx = obj.optString("ctx", ""); + host = obj.optString("host", ""); + crc32 = Long.valueOf(obj.optString("crc32", "0")); + checksum = obj.optString("checksum", ""); + offset = obj.optInt("offset", 0); + } } diff --git a/src/com/qiniu/resumableio/ResumableClient.java b/src/com/qiniu/resumableio/ResumableClient.java index b6a7da63d..4cc141232 100644 --- a/src/com/qiniu/resumableio/ResumableClient.java +++ b/src/com/qiniu/resumableio/ResumableClient.java @@ -18,12 +18,12 @@ import java.util.Map; public class ResumableClient extends Client { - String mUpToken; - int CHUNK_SIZE = 256 * 1024; - int BLOCK_SIZE = 4 * 1024 * 1024; + String mUpToken; + int CHUNK_SIZE = 256 * 1024; + int BLOCK_SIZE = 4 * 1024 * 1024; public ResumableClient(HttpClient client, String uptoken) { super(client); - mUpToken = uptoken; + mUpToken = uptoken; } @Override @@ -34,76 +34,102 @@ protected HttpResponse roundtrip(HttpPost httpPost) throws IOException { return super.roundtrip(httpPost); } - public ICancel[] putblock(final InputStreamAt input, final PutRet putRet, long offset, final JSONObjectRet callback) { - final long writeNeed = Math.min(input.length()-offset, BLOCK_SIZE); - final ICancel[] canceler = new ICancel[] {null}; - JSONObjectRet ret = new JSONObjectRet() { - @Override - public void onSuccess(JSONObject obj) { - putRet.parse(obj); - if (putRet.offset == writeNeed) { - callback.onSuccess(obj); - return; - } - canceler[0] = bput(putRet.host, input, putRet.ctx, putRet.offset, this); - } + public ICancel[] putblock(final InputStreamAt input, final PutRet putRet, final long offset, final JSONObjectRet callback) { + final int writeNeed = (int) Math.min(input.length()-offset, BLOCK_SIZE); + final ICancel[] canceler = new ICancel[] {null}; + final long[] wrote = new long[] {0, 0}; + final long[] crc32 = new long[] {0}; + JSONObjectRet ret = new JSONObjectRet() { + @Override + public void onSuccess(JSONObject obj) { + if (writeNeed < BLOCK_SIZE) { + int c = 1; + c = 2; + } + PutRet pr = new PutRet(); + pr.parse(obj); + if (crc32[0] != pr.crc32) { + putblock(input, putRet, offset, callback); +// callback.onFailure(new Exception("not match")); + return; + } + putRet.parse(obj); + wrote[1] += wrote[0]; + wrote[0] = 0; + if (putRet.offset == writeNeed) { + callback.onSuccess(obj); + return; + } + int remainLength = Math.min((int) (input.length() - offset - putRet.offset), CHUNK_SIZE); + crc32[0] = input.getCrc32(offset+putRet.offset, remainLength); + canceler[0] = bput(putRet.host, input, putRet.ctx, offset, putRet.offset, remainLength, this); + } - @Override - public void onProcess(long current, long total) { - callback.onProcess(current, writeNeed); - } + @Override + public void onProcess(long current, long total) { + wrote[0] = current; + callback.onProcess(wrote[0]+wrote[1], writeNeed); + } - @Override - public void onFailure(Exception ex) { - callback.onFailure(ex); - } - }; - if (putRet.isInvalid()) { - canceler[0] = mkblk(input, offset, ret); - } else { - canceler[0] = bput(putRet.host, input, putRet.ctx, putRet.offset, ret); - } - return canceler; - } + @Override + public void onFailure(Exception ex) { + callback.onFailure(ex); + } + }; + if (putRet.isInvalid()) { + int chunkSize = Math.min(writeNeed, CHUNK_SIZE); + crc32[0] = input.getCrc32(offset, chunkSize); + canceler[0] = mkblk(input, offset, writeNeed, chunkSize, ret); + } else { + int remainLength = Math.min((int) (input.length() - offset - putRet.offset), CHUNK_SIZE); + crc32[0] = input.getCrc32(offset+putRet.offset, remainLength); + canceler[0] = bput(putRet.host, input, putRet.ctx, offset, putRet.offset, remainLength, ret); + } + return canceler; + } - public ICancel mkblk(InputStreamAt input, long offset, CallRet ret) { - int remainLength = Math.min((int) (input.length() - offset), BLOCK_SIZE); - String url = Conf.UP_HOST + "/mkblk/" + remainLength; - remainLength = Math.min((int) (input.length()-offset), CHUNK_SIZE); - return call(url, input.toHttpEntity(offset, remainLength, ret), ret); - } + public ICancel mkblk(InputStreamAt input, long offset, int blockSize, int chunkSize, CallRet ret) { + int remainLength = blockSize; + String url = Conf.UP_HOST + "/mkblk/" + remainLength; + ClientExecuter client = makeClientExecuter(); + call(client, url, input.toHttpEntity(offset, chunkSize, client), ret); + return client; + } - public ICancel bput(String host, InputStreamAt input, String ctx, long offset, CallRet ret) { - int remainLength = Math.min((int) (input.length() - offset), CHUNK_SIZE); - String url = host + "/bput/" + ctx + "/" + offset; - return call(url, input.toHttpEntity(offset, remainLength, ret), ret); - } + public ICancel bput(String host, InputStreamAt input, String ctx, long blockOffset, long offset, int writeLength, CallRet ret) { + String url = host + "/bput/" + ctx + "/" + offset; + ClientExecuter client = makeClientExecuter(); + call(client, url, input.toHttpEntity(blockOffset+offset, writeLength, client), ret); + return client; + } - public ICancel mkfile(String key, long fsize, String mimeType, Map params, String ctxs, CallRet ret) { - String url = Conf.UP_HOST + "/mkfile/" + fsize; - if (mimeType != null) { - url += "/mimeType/" + encode(mimeType); - } - if (key != null) { - url += "/key/" + encode(key); - } - if (params != null && params.size() > 0) { - for (Map.Entry a: params.entrySet()) { - url += "/" + a.getKey() + "/" + encode(a.getValue()); - } - } - StringEntity se; - try { - se = new StringEntity(ctxs); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - ret.onFailure(e); - return null; - } - return call(url, se, ret); - } + public ICancel mkfile(String key, long fsize, String mimeType, Map params, String ctxs, CallRet ret) { + String url = Conf.UP_HOST + "/mkfile/" + fsize; + if (mimeType != null) { + url += "/mimeType/" + encode(mimeType); + } + if (key != null) { + url += "/key/" + encode(key); + } + if (params != null && params.size() > 0) { + for (Map.Entry a: params.entrySet()) { + url += "/" + a.getKey() + "/" + encode(a.getValue()); + } + } + StringEntity se; + try { + se = new StringEntity(ctxs); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + ret.onFailure(e); + return null; + } + ClientExecuter client = makeClientExecuter(); + call(client, url, se, ret); + return client; + } - public String encode(String data) { - return Base64.encodeToString(data.getBytes(), Base64.URL_SAFE | Base64.NO_WRAP); - } + public String encode(String data) { + return Base64.encodeToString(data.getBytes(), Base64.URL_SAFE | Base64.NO_WRAP); + } } diff --git a/src/com/qiniu/resumableio/ResumableIO.java b/src/com/qiniu/resumableio/ResumableIO.java index 71a1550c5..c14f95cd3 100644 --- a/src/com/qiniu/resumableio/ResumableIO.java +++ b/src/com/qiniu/resumableio/ResumableIO.java @@ -1,6 +1,7 @@ package com.qiniu.resumableio; import android.content.Context; +import android.database.Cursor; import android.net.Uri; import com.qiniu.auth.Client; import com.qiniu.auth.JSONObjectRet; @@ -8,128 +9,196 @@ import com.qiniu.utils.InputStreamAt; import org.json.JSONObject; -import java.io.FileNotFoundException; +import java.io.File; +import java.net.URI; +import java.net.URISyntaxException; import java.util.HashMap; public class ResumableIO { - ResumableClient mClient; - private int BLOCK_SIZE = 4 * 1024 * 1024; - private int CHUNK_SIZE = 256 * 1024; - private static int atomicId = 0; - private static HashMap idCancels = new HashMap(); - - public ResumableIO(String uptoken) { - mClient = new ResumableClient(Client.getMultithreadClient(), uptoken); - } - public ResumableIO(ResumableClient client) { - mClient = client; - } - - private synchronized Integer newTask(ICancel[][] c) { - idCancels.put(atomicId, c); - return atomicId++; - } - - public int put(final String key, final InputStreamAt input, final PutExtra extra, final JSONObjectRet ret) { - final int blkCount = (int) (input.length() / BLOCK_SIZE) + 1; - if (extra.processes == null) { - extra.processes = new PutRet[blkCount]; - for (int i=0; i 0) { - ctxes = ctxes.substring(1); - } - mClient.mkfile(key, input.length(), extra.mimeType, extra.params, ctxes, ret); - } - } - @Override - public void onProcess(long current, long total) { - uploaded[index] = current; - current = 0; - for (long c: uploaded) { current += c; } - ret.onProcess(current, input.length()); - } - @Override - public void onFailure(Exception ex) { - if (ex.getMessage().contains("Unauthorization")) { - ret.onFailure(ex); - return; - } - retryTime--; - if (retryTime <= 0) { - ret.onFailure(ex); - return; - } - mClient.putblock(input, process, startPos, this); - } - }); - } - - return newTask(cancelers); - } - - public int putFile(Context mContext, String key, Uri uri, PutExtra extra, final JSONObjectRet ret) { - final InputStreamAt isa; - try { - isa = InputStreamAt.fromInputStream(mContext, mContext.getContentResolver().openInputStream(uri)); - } catch (FileNotFoundException e) { - ret.onFailure(e); - return -1; - } - - return put(key, isa, extra, new JSONObjectRet() { - @Override - public void onSuccess(JSONObject obj) { - isa.close(); - ret.onSuccess(obj); - } - - @Override - public void onProcess(long current, long total) { - ret.onProcess(current, total); - } - - @Override - public void onFailure(Exception ex) { - isa.close(); - ret.onFailure(ex); - } - }); - } - - public synchronized static void stop(int id) { - ICancel[][] c = idCancels.get(id); - for (ICancel[] cc: c) { - cc[0].cancel(true); - } - idCancels.remove(c); - } - - public static int put(String uptoken, String key, InputStreamAt isa, PutExtra extra, JSONObjectRet ret) { - return new ResumableIO(new ResumableClient(Client.getMultithreadClient(), uptoken)).put(key, isa, extra, ret); - } - - public static int putFile(Context mContext, String uptoken, String key, Uri uri, PutExtra extra, JSONObjectRet ret) { - return new ResumableIO(new ResumableClient(Client.getMultithreadClient(), uptoken)).putFile(mContext, key, uri, extra, ret); - } + ResumableClient mClient; + private int BLOCK_SIZE = 4 * 1024 * 1024; + private int CHUNK_SIZE = 256 * 1024; + private static int atomicId = 0; + static HashMap idCancels = new HashMap(); + + public ResumableIO(String uptoken) { + mClient = new ResumableClient(Client.getMultithreadClient(), uptoken); + } + public ResumableIO(ResumableClient client) { + mClient = client; + } + + private synchronized Integer newTask(ICancel c) { + idCancels.put(atomicId, c); + return atomicId++; + } + + private synchronized void removeTask(Integer id) { + idCancels.remove(id); + } + + public int putAndClose(final String key, final InputStreamAt input, final PutExtra extra, final JSONObjectRet ret) { + return put(key, input, extra, new JSONObjectRet() { + @Override + public void onSuccess(JSONObject obj) { + input.close(); + ret.onSuccess(obj); + } + + @Override + public void onProcess(long current, long total) { + ret.onProcess(current, total); + } + + @Override + public void onPause(Object tag) { + ret.onPause(tag); + } + + @Override + public void onFailure(Exception ex) { + input.close(); + ret.onFailure(ex); + } + }); + } + + public int put(final String key, final InputStreamAt input, final PutExtra extra, final JSONObjectRet ret) { + final int blkCount = (int) (input.length() / BLOCK_SIZE) + 1; + if (extra.processes == null) { + extra.processes = new PutRet[blkCount]; + for (int i=0; i 0) { + ctxes = ctxes.substring(1); + } + removeTask(taskId); + mClient.mkfile(key, input.length(), extra.mimeType, extra.params, ctxes, ret); + } + } + + @Override + public synchronized void onProcess(long current, long total) { + if (failured[0]) return; + uploaded[index] = current; + current = 0; + for (long c: uploaded) { current += c; } + ret.onProcess(current, input.length()); + } + + @Override + public void onFailure(Exception ex) { + retryTime--; + if (retryTime <= 0 || (ex.getMessage() != null && ex.getMessage().contains("Unauthorized"))) { + removeTask(taskId); + failured[0] = true; + ret.onFailure(ex); + return; + } + if (ex.getMessage() != null && ex.getMessage().contains("invalid BlockCtx")) { + uploaded[index] = 0; + extra.processes[index] = new PutRet(); + } + cancelers[index] = mClient.putblock(input, extra.processes[index], startPos, this); + } + }); + } + + return taskId; + } + + public int putFile(String key, File file, PutExtra extra, final JSONObjectRet ret) { + final InputStreamAt isa = InputStreamAt.fromFile(file); + return putAndClose(key, isa, extra, ret); + } + + public int putFile(Context mContext, String key, Uri uri, PutExtra extra, final JSONObjectRet ret) { + if (!uri.toString().startsWith("file")) { + uri = convertFileUri(mContext, uri); + } + File file = null; + try { + file = new File(new URI(uri.toString())); + } catch (URISyntaxException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + ret.onFailure(e); + return -1; + } + if (!file.exists()) { + ret.onFailure(new Exception("not exist")); + return -1; + } + InputStreamAt isa = InputStreamAt.fromFile(file); + return putAndClose(key, isa, extra, ret); + } + + public static Uri convertFileUri(Context mContext, Uri uri) { + String filePath = null; + if (uri != null && "content".equals(uri.getScheme())) { + Cursor cursor = mContext.getContentResolver().query(uri, new String[] { android.provider.MediaStore.Images.ImageColumns.DATA }, null, null, null); + cursor.moveToFirst(); + filePath = cursor.getString(0); + cursor.close(); + } + else { + filePath = uri.getPath(); + } + return Uri.parse("file://" + filePath); + } + + public synchronized static void stop(int id) { + ICancel c = idCancels.get(id); + if (c == null) return; + c.cancel(true); + idCancels.remove(id); + } + + public static ResumableIO defaultInstance(String uptoken) { + return new ResumableIO(new ResumableClient(Client.getMultithreadClient(), uptoken)); + } + + public static int putAndClose(String uptoken, String key, InputStreamAt isa, PutExtra extra, JSONObjectRet ret) { + return ResumableIO.defaultInstance(uptoken).putAndClose(uptoken, key, isa, extra, ret); + } + + public static int put(String uptoken, String key, InputStreamAt isa, PutExtra extra, JSONObjectRet ret) { + return ResumableIO.defaultInstance(uptoken).put(key, isa, extra, ret); + } + + public static int putFile(Context mContext, String uptoken, String key, Uri uri, PutExtra extra, JSONObjectRet ret) { + return ResumableIO.defaultInstance(uptoken).putFile(mContext, key, uri, extra, ret); + } } From 13f2cbc750ccab491652d54a0378c86d5650904c Mon Sep 17 00:00:00 2001 From: Cheney Date: Wed, 4 Sep 2013 15:11:28 +0800 Subject: [PATCH 09/16] update io --- src/com/qiniu/io/IO.java | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/com/qiniu/io/IO.java b/src/com/qiniu/io/IO.java index d07dd16ba..07ae92256 100644 --- a/src/com/qiniu/io/IO.java +++ b/src/com/qiniu/io/IO.java @@ -7,6 +7,7 @@ import com.qiniu.conf.Conf; import com.qiniu.utils.InputStreamAt; import com.qiniu.utils.MultipartEntity; +import org.apache.http.client.HttpClient; import org.json.JSONObject; import java.io.FileNotFoundException; @@ -28,15 +29,15 @@ private static Client defaultClient() { } return mClient; } - +哈哈 /** * 上传二进制 * * @param uptoken 用于上传的验证信息 - * @param key 键值名, UNDEFINED_KEY 表示自动生成key - * @param isa 二进制数据 + * @param key 键值名, UNDEFINED_KEY 表示自动生成key + * @param isa 二进制数据 * @param extra 上传参数 - * @param ret 回调函数 + * @param ret 回调函数 */ public static void put(String uptoken, String key, InputStreamAt isa, PutExtra extra, JSONObjectRet ret) { @@ -52,24 +53,25 @@ public static void put(String uptoken, String key, InputStreamAt isa, PutExtra e ret.onFailure(new Exception("uptoken not specify")); return; } - + if (extra.checkCrc == PutExtra.AUTO_CRC32) { extra.crc32 = isa.crc32(); } if (extra.checkCrc != PutExtra.UNUSE_CRC32) { m.addField("crc32", extra.crc32 + ""); } - + for (Map.Entry i: extra.params.entrySet()) { m.addField(i.getKey(), i.getValue()); } m.addField("token", uptoken); m.addFile("file", extra.mimeType, key, isa); - m.setProcessNotify(ret); - + m.setProcessNotify(ret); - defaultClient().call(Conf.UP_HOST, m, ret); + Client client = defaultClient(); + Client.ClientExecuter executer = client.makeClientExecuter(); + client.call(executer, Conf.UP_HOST, m, ret); } /** @@ -99,12 +101,12 @@ public void onSuccess(JSONObject obj) { ret.onSuccess(obj); } - @Override - public void onProcess(long current, long total) { - ret.onProcess(current, total); - } + @Override + public void onProcess(long current, long total) { + ret.onProcess(current, total); + } - @Override + @Override public void onFailure(Exception ex) { isa.close(); ret.onFailure(ex); @@ -112,3 +114,4 @@ public void onFailure(Exception ex) { }); } } + From 1d7ba1fef5e4dcbb3640f80edc8c26e99c182e58 Mon Sep 17 00:00:00 2001 From: Cheney Date: Wed, 4 Sep 2013 15:11:37 +0800 Subject: [PATCH 10/16] update client --- src/com/qiniu/auth/CallRet.java | 6 +-- src/com/qiniu/auth/Client.java | 88 ++++++++++++++++++--------------- 2 files changed, 52 insertions(+), 42 deletions(-) diff --git a/src/com/qiniu/auth/CallRet.java b/src/com/qiniu/auth/CallRet.java index 8072ad653..a26343f1d 100644 --- a/src/com/qiniu/auth/CallRet.java +++ b/src/com/qiniu/auth/CallRet.java @@ -5,6 +5,6 @@ public abstract class CallRet implements IOnProcess { public abstract void onSuccess(byte[] body); public abstract void onFailure(Exception ex); - public void onProcess(long current, long total){} - public void onPause(){} -} \ No newline at end of file + public void onProcess(long current, long total){} + public void onPause(Object tag){} +} diff --git a/src/com/qiniu/auth/Client.java b/src/com/qiniu/auth/Client.java index cdfb3df90..175d4a981 100644 --- a/src/com/qiniu/auth/Client.java +++ b/src/com/qiniu/auth/Client.java @@ -16,46 +16,46 @@ import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; import org.apache.http.params.BasicHttpParams; import org.apache.http.params.HttpParams; +import org.apache.http.protocol.HttpContext; import org.apache.http.util.EntityUtils; import java.io.IOException; public class Client { - + protected HttpClient mClient; - + public Client(HttpClient client) { mClient = client; } - public ICancel call(String url, CallRet ret) { - HttpPost httppost = new HttpPost(url); - return execute(httppost, ret); - } - - public ICancel call(String url, HttpEntity entity, CallRet ret) { - Header header = entity.getContentType(); - String contentType = "application/octet-stream"; - if (header != null) { - contentType = header.getValue(); - } - return call(url, contentType, entity, ret); + public void call(ClientExecuter client, String url, HttpEntity entity, CallRet ret) { + Header header = entity.getContentType(); + String contentType = "application/octet-stream"; + if (header != null) { + contentType = header.getValue(); + } + call(client, url, contentType, entity, ret); } - public ICancel call(String url, String contentType, HttpEntity entity, CallRet ret) { + public void call(ClientExecuter client, String url, String contentType, HttpEntity entity, CallRet ret) { HttpPost httppost = new HttpPost(url); httppost.setEntity(entity); if (contentType != null) { httppost.setHeader("Content-Type", contentType); } - return execute(httppost, ret); + execute(client, httppost, ret); } - protected ClientExecuter execute(HttpPost httpPost, CallRet ret) { - ClientExecuter client = new ClientExecuter(); - client.execute(httpPost, ret); - return client; + public ClientExecuter makeClientExecuter() { + return new ClientExecuter(); + } + + protected ClientExecuter execute(ClientExecuter client, HttpPost httpPost, final CallRet ret) { + client.setup(httpPost, ret); + client.execute(); + return client; } protected HttpResponse roundtrip(HttpPost httpPost) throws IOException { @@ -63,30 +63,35 @@ protected HttpResponse roundtrip(HttpPost httpPost) throws IOException { return mClient.execute(httpPost); } - class ClientExecuter extends AsyncTask implements ICancel { - HttpPost httpPost; - CallRet ret; + public class ClientExecuter extends AsyncTask implements ICancel { + HttpPost mHttpPost; + CallRet mRet; + public void setup(HttpPost httpPost, CallRet ret) { + mHttpPost = httpPost; + mRet = ret; + } + public void upload(long current, long total) { + publishProgress(current, total); + } @Override protected Object doInBackground(Object... objects) { - httpPost = (HttpPost) objects[0]; - ret = (CallRet) objects[1]; try { - HttpResponse resp = roundtrip(httpPost); - int statusCode = resp.getStatusLine().getStatusCode(); - if (statusCode == 401) { // android 2.3 will not response - return new Exception(resp.getStatusLine().getReasonPhrase()); - } + HttpResponse resp = roundtrip(mHttpPost); + int statusCode = resp.getStatusLine().getStatusCode(); + if (statusCode == 401) { // android 2.3 will not response + return new Exception(resp.getStatusLine().getReasonPhrase()); + } byte[] data = EntityUtils.toByteArray(resp.getEntity()); if (statusCode / 100 != 2) { - if (data.length == 0) { - String xlog = resp.getFirstHeader("X-Log").getValue(); - if (xlog.length() > 0) { - return new Exception(xlog); - } - return new Exception(resp.getStatusLine().getReasonPhrase()); - } + if (data.length == 0) { + String xlog = resp.getFirstHeader("X-Log").getValue(); + if (xlog.length() > 0) { + return new Exception(xlog); + } + return new Exception(resp.getStatusLine().getReasonPhrase()); + } return new Exception(new String(data)); } return data; @@ -96,13 +101,18 @@ protected Object doInBackground(Object... objects) { } } + @Override + protected void onProgressUpdate(Object... values) { + mRet.onProcess((Long) values[0], (Long) values[1]); + } + @Override protected void onPostExecute(Object o) { if (o instanceof Exception) { - ret.onFailure((Exception) o); + mRet.onFailure((Exception) o); return; } - ret.onSuccess((byte[]) o); + mRet.onSuccess((byte[]) o); } }; From a9a79e89bb970c1a519efc7ce02f46c2827d8e88 Mon Sep 17 00:00:00 2001 From: Cheney Date: Wed, 4 Sep 2013 15:11:48 +0800 Subject: [PATCH 11/16] add Resumable demo --- AndroidManifest.xml | 6 ++ res/layout/resumable.xml | 50 +++++++++ res/values/strings.xml | 1 + src/com/qiniu/demo/MyResumableActivity.java | 110 ++++++++++++++++++++ 4 files changed, 167 insertions(+) create mode 100644 res/layout/resumable.xml create mode 100644 src/com/qiniu/demo/MyResumableActivity.java diff --git a/AndroidManifest.xml b/AndroidManifest.xml index 54c214f0c..835918e56 100644 --- a/AndroidManifest.xml +++ b/AndroidManifest.xml @@ -5,10 +5,16 @@ android:versionName="1.0"> + + + + diff --git a/res/layout/resumable.xml b/res/layout/resumable.xml new file mode 100644 index 000000000..f60ea3fde --- /dev/null +++ b/res/layout/resumable.xml @@ -0,0 +1,50 @@ + + +