Skip to content

Commit

Permalink
add features to storage module(android) (#1296)
Browse files Browse the repository at this point in the history 
* storage enhance. add LRU strategy to remove item

* add storage compatible function

* add test

* add note/remove space

* code defense for storage

* add log

* update

* update
  • Loading branch information
@Rowandjj @sospartan
Rowandjj authored and sospartan committed Sep 26, 2016
1 parent 859ef3b commit f60c292
Show file tree
Hide file tree
Showing 6 changed files with 204 additions and 22 deletions.
108 changes: 98 additions & 10 deletions android/sdk/src/main/java/com/taobao/weex/appfram/storage/DefaultWXStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,13 +204,16 @@
*/
package com.taobao.weex.appfram.storage;

import android.content.ContentValues;
import android.content.Context;
import android.database.Cursor;
import android.database.sqlite.SQLiteFullException;
import android.database.sqlite.SQLiteStatement;

import com.taobao.weex.utils.WXLogUtils;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
Expand Down Expand Up @@ -239,7 +242,10 @@ public void setItem(final String key, final String value, final OnResultReceived
execute(new Runnable() {
@Override
public void run() {
Map<String, Object> data = StorageResultHandler.setItemResult(performSetItem(key, value));
Map<String, Object> data = StorageResultHandler.setItemResult(performSetItem(key, value, false, true));
if(listener == null){
return;
}
listener.onReceived(data);
}
});
Expand All @@ -251,6 +257,9 @@ public void getItem(final String key, final OnResultReceivedListener listener) {
@Override
public void run() {
Map<String, Object> data = StorageResultHandler.getItemResult(performGetItem(key));
if(listener == null){
return;
}
listener.onReceived(data);
}
});
Expand All @@ -262,6 +271,9 @@ public void removeItem(final String key, final OnResultReceivedListener listener
@Override
public void run() {
Map<String, Object> data = StorageResultHandler.removeItemResult(performRemoveItem(key));
if(listener == null){
return;
}
listener.onReceived(data);
}
});
Expand All @@ -273,6 +285,9 @@ public void length(final OnResultReceivedListener listener) {
@Override
public void run() {
Map<String, Object> data = StorageResultHandler.getLengthResult(performGetLength());
if(listener == null){
return;
}
listener.onReceived(data);
}
});
Expand All @@ -284,6 +299,23 @@ public void getAllKeys(final OnResultReceivedListener listener) {
@Override
public void run() {
Map<String, Object> data = StorageResultHandler.getAllkeysResult(performGetAllKeys());
if(listener == null){
return;
}
listener.onReceived(data);
}
});
}

@Override
public void setItemPersistent(final String key, final String value, final OnResultReceivedListener listener) {
execute(new Runnable() {
@Override
public void run() {
Map<String, Object> data = StorageResultHandler.setItemResult(performSetItem(key, value, true, true));
if(listener == null){
return;
}
listener.onReceived(data);
}
});
Expand All @@ -294,24 +326,73 @@ public void close() {
mDatabaseSupplier.closeDatabase();
}


private boolean performSetItem(String key, String value) {
String sql = "INSERT OR REPLACE INTO " + WXSQLiteOpenHelper.TABLE_STORAGE + " VALUES (?,?);";
private boolean performSetItem(String key, String value, boolean isPersistent, boolean allowRetryWhenFull) {
WXLogUtils.d(WXSQLiteOpenHelper.TAG_STORAGE,"set k-v to storage(key:"+ key + ",value:"+ value+",isPersistent:"+isPersistent+",allowRetry:"+allowRetryWhenFull+")");
String sql = "INSERT OR REPLACE INTO " + WXSQLiteOpenHelper.TABLE_STORAGE + " VALUES (?,?,?,?);";
SQLiteStatement statement = mDatabaseSupplier.getDatabase().compileStatement(sql);
String timeStamp = WXSQLiteOpenHelper.sDateFormatter.format(new Date());
try {
statement.clearBindings();
statement.bindString(1, key);
statement.bindString(2, value);
statement.bindString(3, timeStamp);
statement.bindLong(4, isPersistent ? 1 : 0);
statement.execute();
return true;
} catch (Exception e) {
WXLogUtils.e("DefaultWXStorage", e.getMessage());
WXLogUtils.e(WXSQLiteOpenHelper.TAG_STORAGE,"DefaultWXStorage occurred an exception when execute setItem :" + e.getMessage());
if(e instanceof SQLiteFullException){
if(allowRetryWhenFull && trimToSize()){
//try again
//setItem/setItemPersistent method only allow try once when occurred a sqliteFullException.
WXLogUtils.d(WXSQLiteOpenHelper.TAG_STORAGE,"retry set k-v to storage(key:"+key+",value:"+value+")");
return performSetItem(key,value,isPersistent,false);
}
}

return false;
} finally {
statement.close();
}
}

/**
* remove 10% of total record(at most) ordered by timestamp.
* */
private boolean trimToSize(){
List<String> toEvict = new ArrayList<>();
int num = 0;
Cursor c = mDatabaseSupplier.getDatabase().query(WXSQLiteOpenHelper.TABLE_STORAGE, new String[]{WXSQLiteOpenHelper.COLUMN_KEY,WXSQLiteOpenHelper.COLUMN_PERSISTENT}, null, null, null, null, WXSQLiteOpenHelper.COLUMN_TIMESTAMP+" ASC");
try {
int evictSize = c.getCount() / 10;
while (c.moveToNext()) {
String key = c.getString(c.getColumnIndex(WXSQLiteOpenHelper.COLUMN_KEY));
boolean persistent = c.getInt(c.getColumnIndex(WXSQLiteOpenHelper.COLUMN_PERSISTENT)) == 1;
if(!persistent && key != null){
num++;
toEvict.add(key);
if(num == evictSize){
break;
}
}
}
} catch (Exception e) {
WXLogUtils.e(WXSQLiteOpenHelper.TAG_STORAGE,"DefaultWXStorage occurred an exception when execute trimToSize:"+e.getMessage());
} finally {
c.close();
}

if(num <= 0){
return false;
}

for(String key : toEvict){
performRemoveItem(key);
}
WXLogUtils.e(WXSQLiteOpenHelper.TAG_STORAGE,"remove "+ num +" items by lru");
return true;
}

private String performGetItem(String key) {
Cursor c = mDatabaseSupplier.getDatabase().query(WXSQLiteOpenHelper.TABLE_STORAGE,
new String[]{WXSQLiteOpenHelper.COLUMN_VALUE},
Expand All @@ -320,12 +401,18 @@ private String performGetItem(String key) {
null, null, null);
try {
if (c.moveToNext()) {
ContentValues values = new ContentValues();
//update timestamp
values.put(WXSQLiteOpenHelper.COLUMN_TIMESTAMP,WXSQLiteOpenHelper.sDateFormatter.format(new Date()));
int updateResult = mDatabaseSupplier.getDatabase().update(WXSQLiteOpenHelper.TABLE_STORAGE,values,WXSQLiteOpenHelper.COLUMN_KEY+"= ?",new String[]{key});

WXLogUtils.d(WXSQLiteOpenHelper.TAG_STORAGE,"update timestamp "+ (updateResult == 1 ? "success" : "failed") + " for operation [getItem(key = "+key+")]" );
return c.getString(c.getColumnIndex(WXSQLiteOpenHelper.COLUMN_VALUE));
} else {
return null;
}
} catch (Exception e) {
WXLogUtils.e("DefaultWXStorage", e.getMessage());
WXLogUtils.e(WXSQLiteOpenHelper.TAG_STORAGE,"DefaultWXStorage occurred an exception when execute getItem:"+e.getMessage());
return null;
} finally {
c.close();
Expand All @@ -338,7 +425,9 @@ private boolean performRemoveItem(String key) {
count = mDatabaseSupplier.getDatabase().delete(WXSQLiteOpenHelper.TABLE_STORAGE,
WXSQLiteOpenHelper.COLUMN_KEY + "=?",
new String[]{key});
} finally {
} catch (Exception e) {
WXLogUtils.e(WXSQLiteOpenHelper.TAG_STORAGE,"DefaultWXStorage occurred an exception when execute removeItem:" + e.getMessage());
return false;
}
return count == 1;
}
Expand All @@ -349,7 +438,7 @@ private long performGetLength() {
try {
return statement.simpleQueryForLong();
} catch (Exception e) {
WXLogUtils.e("DefaultWXStorage", e.getMessage());
WXLogUtils.e(WXSQLiteOpenHelper.TAG_STORAGE,"DefaultWXStorage occurred an exception when execute getLength:"+e.getMessage());
return 0;
} finally {
statement.close();
Expand All @@ -365,12 +454,11 @@ private List<String> performGetAllKeys() {
}
return result;
} catch (Exception e) {
WXLogUtils.e("DefaultWXStorage", e.getMessage());
WXLogUtils.e(WXSQLiteOpenHelper.TAG_STORAGE,"DefaultWXStorage occurred an exception when execute getAllKeys:"+e.getMessage());
return result;
} finally {
c.close();
}
}


}
2 changes: 1 addition & 1 deletion android/sdk/src/main/java/com/taobao/weex/appfram/storage/IWXStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,5 +214,5 @@ interface IWXStorage {
public void removeItem(String key,@Nullable JSCallback callback);
public void length(@Nullable JSCallback callback);
public void getAllKeys(@Nullable JSCallback callback);

public void setItemPersistent(String key, String value, @Nullable JSCallback callback);
}
2 changes: 2 additions & 0 deletions android/sdk/src/main/java/com/taobao/weex/appfram/storage/IWXStorageAdapter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,8 @@ public interface IWXStorageAdapter {

void getAllKeys(OnResultReceivedListener listener);

void setItemPersistent(String key, String value, OnResultReceivedListener listener);

void close();

/**
Expand Down
78 changes: 72 additions & 6 deletions android/sdk/src/main/java/com/taobao/weex/appfram/storage/WXSQLiteOpenHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,12 +210,19 @@

import com.taobao.weex.utils.WXLogUtils;

import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;

public class WXSQLiteOpenHelper extends SQLiteOpenHelper {

private static final String DATABASE_NAME = "WXStorage";
private static final int DATABASE_VERSION = 1;
private static final int DATABASE_VERSION = 2;
static final String TAG_STORAGE = "weex_storage";

private long mMaximumDatabaseSize = 5 * 10 * 1024 * 1024L;//50mb
static SimpleDateFormat sDateFormatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.getDefault());

private long mMaximumDatabaseSize = 5L * 1024L * 1024L;

private static WXSQLiteOpenHelper sInstance;

Expand All @@ -226,12 +233,19 @@ public class WXSQLiteOpenHelper extends SQLiteOpenHelper {
static final String TABLE_STORAGE = "default_wx_storage";
static final String COLUMN_KEY = "key";
static final String COLUMN_VALUE = "value";
static final String COLUMN_TIMESTAMP = "timestamp";
static final String COLUMN_PERSISTENT = "persistent";


private static final String STATEMENT_CREATE_TABLE = "CREATE TABLE IF NOT EXISTS " + TABLE_STORAGE + " ("
+ COLUMN_KEY
+ " TEXT PRIMARY KEY,"
+ COLUMN_VALUE
+ " TEXT NOT NULL"
+ " TEXT NOT NULL,"
+ COLUMN_TIMESTAMP
+ " TEXT NOT NULL,"
+ COLUMN_PERSISTENT
+ " INTEGER DEFAULT 0"
+ ")";


Expand All @@ -242,7 +256,7 @@ private WXSQLiteOpenHelper(Context context) {

public static WXSQLiteOpenHelper getInstance(Context context) {
if (context == null) {
WXLogUtils.e("can not get context instance...");
WXLogUtils.e(TAG_STORAGE,"can not get context instance...");
return null;
}
if (sInstance == null) {
Expand All @@ -261,11 +275,63 @@ public void onCreate(SQLiteDatabase db) {
db.execSQL(STATEMENT_CREATE_TABLE);
}


/**
* version 1:
*
* ----------------
* | key | value |
* ---------------
*
* version 2:
*
* ----------------------------------------
* | key | value | timestamp | persistent |
* ----------------------------------------
**/
@Override
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
if (oldVersion != newVersion) {
deleteDB();
onCreate(db);
if(newVersion == 2 && oldVersion == 1){
WXLogUtils.d(TAG_STORAGE,"storage is updating from version "+oldVersion+" to version "+newVersion);
boolean updateResult = true;
try {
long start = System.currentTimeMillis();

db.beginTransaction();
// update table structure
String SQL_ADD_COLUMN_TIMESTAMP = "ALTER TABLE "+TABLE_STORAGE+" ADD COLUMN "+COLUMN_TIMESTAMP+" TEXT;";
WXLogUtils.d(TAG_STORAGE,"exec sql : "+ SQL_ADD_COLUMN_TIMESTAMP);
db.execSQL(SQL_ADD_COLUMN_TIMESTAMP);

String SQL_ADD_COLUMN_PERSISTENT = "ALTER TABLE "+TABLE_STORAGE+" ADD COLUMN "+COLUMN_PERSISTENT+" INTEGER;";
WXLogUtils.d(TAG_STORAGE,"exec sql : "+ SQL_ADD_COLUMN_PERSISTENT);
db.execSQL(SQL_ADD_COLUMN_PERSISTENT);

// update timestamp & persistent
String SQL_UPDATE_TABLE = "UPDATE "+TABLE_STORAGE+" SET "+ COLUMN_TIMESTAMP+" = '"+sDateFormatter.format(new Date())+"' , "+ COLUMN_PERSISTENT +" = 0";
WXLogUtils.d(TAG_STORAGE,"exec sql : "+ SQL_UPDATE_TABLE);
db.execSQL(SQL_UPDATE_TABLE);

db.setTransactionSuccessful();
long time = System.currentTimeMillis() - start;
WXLogUtils.d(TAG_STORAGE,"storage updated success ("+time+"ms)");
}catch (Exception e){
WXLogUtils.d(TAG_STORAGE,"storage updated failed from version "+oldVersion+" to version "+newVersion+","+e.getMessage());
updateResult = false;
}finally {
db.endTransaction();
}
//rollback
if(!updateResult){
WXLogUtils.d(TAG_STORAGE,"storage is rollback,all data will be removed");
deleteDB();
onCreate(db);
}
}else{
deleteDB();
onCreate(db);
}
}
}

Expand Down
23 changes: 22 additions & 1 deletion android/sdk/src/main/java/com/taobao/weex/appfram/storage/WXStorageModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,6 @@

import com.taobao.weex.WXSDKEngine;
import com.taobao.weex.bridge.JSCallback;
import com.taobao.weex.common.WXModule;
import com.taobao.weex.common.WXModuleAnno;

import java.util.Map;
Expand Down Expand Up @@ -337,6 +336,28 @@ public void onReceived(Map<String, Object> data) {
});
}

@Override
public void setItemPersistent(String key, String value, @Nullable final JSCallback callback) {
if (TextUtils.isEmpty(key) || TextUtils.isEmpty(value)) {
StorageResultHandler.handleInvalidParam(callback);
return;
}

IWXStorageAdapter adapter = ability();
if (adapter == null) {
StorageResultHandler.handleNoHandlerError(callback);
return;
}
adapter.setItemPersistent(key, value, new IWXStorageAdapter.OnResultReceivedListener() {
@Override
public void onReceived(Map<String, Object> data) {
if(callback != null){
callback.invoke(data);
}
}
});
}

@Override
public void destroy() {
IWXStorageAdapter adapter = ability();
Expand Down
Loading

0 comments on commit f60c292

Please sign in to comment.