Skip to content

Commit

Permalink
#435 low disk space. this resolves a number of problems with the samp…
Browse files Browse the repository at this point in the history 
…le app + the library not being disk space aware. Also fixes a bug in the sqlitetilewriter's clean up mechanism for trimming. Preferences for developers to turn on and off the debug settings without recompiling the sample app. User selection of cache location and how to work with reconfiguring the cache settings. This kind of highlights that we need to rework osmdroid to have some kind of configuration class to manage all the settings, storage locations and to bounce sqlite writers and more
  • Loading branch information
@spyhunter99
spyhunter99 committed Oct 22, 2016
1 parent 474019f commit bcd59b8
Show file tree
Hide file tree
Showing 5 changed files with 151 additions and 54 deletions.
39 changes: 37 additions & 2 deletions OpenStreetMapViewer/src/main/java/org/osmdroid/MainActivity.java
View file
Expand Up @@ -14,6 +14,7 @@
import android.os.Environment;
import android.preference.PreferenceManager;
import android.support.v4.content.ContextCompat;
import android.util.Log;
import android.view.View;
import android.widget.AdapterView;
import android.widget.ArrayAdapter;
Expand All @@ -27,6 +28,7 @@
import org.osmdroid.samples.SampleWithTilesOverlay;
import org.osmdroid.samples.SampleWithTilesOverlayAndCustomTileSource;
import org.osmdroid.tileprovider.constants.OpenStreetMapTileProviderConstants;
import org.osmdroid.tileprovider.modules.SqlTileWriter;
import org.osmdroid.views.MapView;

import java.io.File;
Expand All @@ -37,6 +39,7 @@

public class MainActivity extends Activity implements AdapterView.OnItemClickListener {

public static final String TAG = "OSM";
/**
* Called when the activity is first created.
*/
Expand All @@ -51,15 +54,47 @@ public void onCreate(final Bundle savedInstanceState) {
}


//cache management starts here
File discoveredBestPath = OpenStreetMapTileProviderConstants.TILE_PATH_BASE;


//grab the current user preferences for debug settings and where to store the tile cache data
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this);
OpenStreetMapTileProviderConstants.TILE_PATH_BASE = new File(prefs.getString("textViewCacheDirectory", discoveredBestPath.getAbsolutePath()));
OpenStreetMapTileProviderConstants.setCachePath(prefs.getString("textViewCacheDirectory", discoveredBestPath.getAbsolutePath()));
OpenStreetMapTileProviderConstants.DEBUGMODE=prefs.getBoolean("checkBoxDebugMode",false);
OpenStreetMapTileProviderConstants.DEBUG_TILE_PROVIDERS=prefs.getBoolean("checkBoxDebugTileProvider",false);
MapView.hardwareAccelerated=prefs.getBoolean("checkBoxHardwareAcceleration",false);

OpenStreetMapTileProviderConstants.TILE_MAX_CACHE_SIZE_BYTES = 16000;

if (Build.VERSION.SDK_INT >= 9) {
//https://github/osmdroid/osmdroid/issues/435
//On startup, we auto set the max cache size to be the current cache size + free disk space
//this reduces the chance of osmdroid completely filling up the storage device

//if the default max cache size is greater than the available free space
//reduce it to 95% of the available free space + the size of the cache
File dbFile = new File(OpenStreetMapTileProviderConstants.TILE_PATH_BASE.getAbsolutePath() + File.separator + SqlTileWriter.DATABASE_FILENAME);
if (dbFile.exists()) {
long cacheSize = dbFile.length();
long freeSpace = OpenStreetMapTileProviderConstants.TILE_PATH_BASE.getFreeSpace();

Log.i(TAG, "Current cache size is " + cacheSize + " free space is " + freeSpace);
if (OpenStreetMapTileProviderConstants.TILE_MAX_CACHE_SIZE_BYTES > (freeSpace + cacheSize)){
OpenStreetMapTileProviderConstants.TILE_MAX_CACHE_SIZE_BYTES = (long)((freeSpace + cacheSize) * 0.95);
OpenStreetMapTileProviderConstants.TILE_TRIM_CACHE_SIZE_BYTES = (long)((freeSpace + cacheSize) * 0.90);
}
} else {
//this is probably the first time running osmdroid
long freeSpace = OpenStreetMapTileProviderConstants.TILE_PATH_BASE.length();
if (OpenStreetMapTileProviderConstants.TILE_MAX_CACHE_SIZE_BYTES > (freeSpace)){
OpenStreetMapTileProviderConstants.TILE_MAX_CACHE_SIZE_BYTES = (long)((freeSpace) * 0.95);
OpenStreetMapTileProviderConstants.TILE_TRIM_CACHE_SIZE_BYTES = (long)((freeSpace) * 0.90);
}
}
}

//cache management ends here


// Generate a ListView with Sample Maps
final ArrayList<String> list = new ArrayList<>();
Expand Down
54 changes: 46 additions & 8 deletions OpenStreetMapViewer/src/main/java/org/osmdroid/PreferenceActivity.java
View file
Expand Up @@ -10,6 +10,7 @@
import android.text.InputType;
import android.text.TextWatcher;
import android.view.View;
import android.widget.ArrayAdapter;
import android.widget.Button;
import android.widget.CheckBox;
import android.widget.EditText;
Expand All @@ -20,6 +21,7 @@
import org.osmdroid.views.MapView;

import java.io.File;
import java.util.List;

/**
* Created by alex on 10/21/16.
Expand Down Expand Up @@ -59,7 +61,7 @@ public void onResume() {
checkBoxDebugMode.setChecked(prefs.getBoolean("checkBoxDebugMode", OpenStreetMapTileProviderConstants.DEBUGMODE));
checkBoxDebugTileProvider.setChecked(prefs.getBoolean("checkBoxDebugTileProvider", OpenStreetMapTileProviderConstants.DEBUG_TILE_PROVIDERS));
checkBoxHardwareAcceleration.setChecked(prefs.getBoolean("checkBoxHardwareAcceleration", MapView.hardwareAccelerated));
textViewCacheDirectory.setText(prefs.getString("textViewCacheDirectory", OpenStreetMapTileProviderConstants.getBasePath().getAbsolutePath()));
textViewCacheDirectory.setText(prefs.getString("textViewCacheDirectory", OpenStreetMapTileProviderConstants.TILE_PATH_BASE.getAbsolutePath()));

}

Expand All @@ -69,7 +71,7 @@ public void onPause() {
OpenStreetMapTileProviderConstants.DEBUGMODE = checkBoxDebugMode.isChecked();
OpenStreetMapTileProviderConstants.DEBUG_TILE_PROVIDERS = checkBoxDebugTileProvider.isChecked();
MapView.hardwareAccelerated = checkBoxHardwareAcceleration.isChecked();
OpenStreetMapTileProviderConstants.TILE_PATH_BASE=new File(textViewCacheDirectory.getText().toString());
OpenStreetMapTileProviderConstants.setCachePath(textViewCacheDirectory.getText().toString());

SharedPreferences.Editor edit = PreferenceManager.getDefaultSharedPreferences(this).edit();
edit.putBoolean("checkBoxDebugMode", checkBoxDebugMode.isChecked());
Expand All @@ -86,25 +88,62 @@ public void onClick(View v) {
case R.id.buttonManualCacheEntry: {
showManualEntry();
}
break;
case R.id.buttonSetCache: {

showPickCacheFromList();
}
break;

}
}

String m_Text = "";
private void showPickCacheFromList() {

AlertDialog.Builder builder = new AlertDialog.Builder(this);
builder.setTitle("Enter Cache Location");

final ArrayAdapter<String> arrayAdapter = new ArrayAdapter<>(this, android.R.layout.select_dialog_singlechoice);

final List<StorageUtils.StorageInfo> storageList = StorageUtils.getStorageList();
for (int i=0; i < storageList.size(); i++) {
if (!storageList.get(i).readonly)
arrayAdapter.add(storageList.get(i).path);
}

builder.setAdapter(arrayAdapter, new DialogInterface.OnClickListener() {
@Override
public void onClick(DialogInterface dialog, int which) {
String item = arrayAdapter.getItem(which);
try {
new File(item + File.separator + "osmdroid" + File.separator + "tiles" + File.separator).mkdirs();
}catch (Exception ex){
ex.printStackTrace();
}
textViewCacheDirectory.setText(item + File.separator + "osmdroid" + File.separator + "tiles");
}
});
builder.setNegativeButton("Cancel", new DialogInterface.OnClickListener() {
@Override
public void onClick(DialogInterface dialog, int which) {
dialog.cancel();
}
});

builder.show();
}


private void showManualEntry() {

AlertDialog.Builder builder = new AlertDialog.Builder(this);
builder.setTitle("Title");
builder.setTitle("Enter Cache Location");

// Set up the input
final EditText input = new EditText(this);
// Specify the type of input expected; this, for example, sets the input as a password, and will mask the text
input.setInputType(InputType.TYPE_CLASS_TEXT | InputType.TYPE_TEXT_FLAG_NO_SUGGESTIONS);
input.setLines(1);
input.setText(textViewCacheDirectory.getText().toString());
input.addTextChangedListener(new TextWatcher() {
@Override
public void beforeTextChanged(CharSequence s, int start, int count, int after) {
Expand Down Expand Up @@ -132,13 +171,12 @@ public void afterTextChanged(Editable s) {
});
builder.setView(input);

// Set up the buttons
// Set up the buttons
builder.setPositiveButton("OK", new DialogInterface.OnClickListener() {
@Override
public void onClick(DialogInterface dialog, int which) {
if (input.getError()==null) {
m_Text = input.getText().toString();
textViewCacheDirectory.setText(m_Text);
textViewCacheDirectory.setText(input.getText().toString());
}
}
});
Expand Down
5 changes: 3 additions & 2 deletions ...src/main/java/org/osmdroid/tileprovider/constants/OpenStreetMapTileProviderConstants.java
View file
Expand Up @@ -6,7 +6,6 @@

import org.osmdroid.tileprovider.LRUMapTileCache;

import android.os.Environment;
import android.util.Log;
import org.osmdroid.api.IMapView;
import org.osmdroid.tileprovider.util.StorageUtils;
Expand Down Expand Up @@ -41,7 +40,9 @@ public static File getBasePath(){
} catch (Exception ex) {
Log.e(IMapView.LOGTAG, "unable to create a nomedia file. downloaded tiles may be visible to the gallery. " + ex.getMessage());
}
}
}


public static boolean DEBUGMODE = false;
public static boolean DEBUG_TILE_PROVIDERS = false;
public static String USER_AGENT="User-Agent";
Expand Down
2 changes: 2 additions & 0 deletions osmdroid-android/src/main/java/org/osmdroid/tileprovider/modules/MapTileDownloader.java
View file
Expand Up @@ -226,6 +226,8 @@ public Drawable loadTile(final MapTileRequestState aState) throws CantContinueEx
final ByteArrayInputStream byteStream = new ByteArrayInputStream(data);

// Save the data to the filesystem cache
//this is the only point in which we insert tiles to the db or local file system.

if (mFilesystemCache != null) {
mFilesystemCache.saveFile(tileSource, tile, byteStream);
byteStream.reset();
Expand Down
105 changes: 63 additions & 42 deletions osmdroid-android/src/main/java/org/osmdroid/tileprovider/modules/SqlTileWriter.java
View file
Expand Up @@ -3,6 +3,7 @@
import android.content.ContentValues;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteFullException;
import android.util.Log;

import org.osmdroid.api.IMapView;
Expand All @@ -19,6 +20,7 @@
import java.util.Date;
import java.util.List;

import static org.osmdroid.tileprovider.modules.DatabaseFileArchive.COLUMN_KEY;
import static org.osmdroid.tileprovider.modules.DatabaseFileArchive.TABLE;

/**
Expand All @@ -33,14 +35,16 @@
* @since 5.1
*/
public class SqlTileWriter implements IFilesystemCache {
public static final String DATABASE_FILENAME = "cache.db";

protected File db_file;
protected SQLiteDatabase db;
final int questimate=8000;
final int questimate=4000;
static boolean hasInited=false;

public SqlTileWriter() {

db_file = new File(OpenStreetMapTileProviderConstants.TILE_PATH_BASE.getAbsolutePath() + File.separator + "cache.db");
db_file = new File(OpenStreetMapTileProviderConstants.TILE_PATH_BASE.getAbsolutePath() + File.separator + DATABASE_FILENAME);
OpenStreetMapTileProviderConstants.TILE_PATH_BASE.mkdirs();

try {
Expand All @@ -57,51 +61,62 @@ public SqlTileWriter() {
final Thread t = new Thread() {
@Override
public void run() {
if (db==null) {
if (OpenStreetMapTileProviderConstants.DEBUGMODE) {
Log.d(IMapView.LOGTAG, "Finished init thread, aborted due to null database reference");
}
return;
}

try {
//run the reaper (remove all old expired tiles)
//keep if now is < expiration date
//delete if now is > expiration date
long now = System.currentTimeMillis();
int rows = db.delete(TABLE, "expires < ?", new String[]{System.currentTimeMillis() + ""});
Log.d(IMapView.LOGTAG, "Local storage cahce purged " + rows + " expired tiles in " + (System.currentTimeMillis() - now) + "ms, cache size is " + db_file.length() + "bytes");

//VACUUM the database
now = System.currentTimeMillis();
//db.execSQL("VACUUM " + DatabaseFileArchive.TABLE + ";");
// Log.d(IMapView.LOGTAG, "VACUUM completed in " + (System.currentTimeMillis()-now) + "ms, cache size is " + db_file.length() + "bytes");
if (db_file.length() > OpenStreetMapTileProviderConstants.TILE_MAX_CACHE_SIZE_BYTES) {
long diff = OpenStreetMapTileProviderConstants.TILE_MAX_CACHE_SIZE_BYTES - db_file.length();
long tilesToKill = diff / questimate;
try {
db.execSQL("DELETE FROM " + TABLE + " ORDER BY expires DESC LIMIT " + tilesToKill);
} catch (Throwable t) {
t.printStackTrace();
}
Log.d(IMapView.LOGTAG, "purge completed in " + (System.currentTimeMillis() - now) + "ms, cache size is " + db_file.length() + "bytes");
}
}catch (Exception ex){
if (OpenStreetMapTileProviderConstants.DEBUGMODE) {
Log.d(IMapView.LOGTAG, "SqliteTileWriter init thread crash, db is probably not available",ex);
}
}

if (OpenStreetMapTileProviderConstants.DEBUGMODE) {
Log.d(IMapView.LOGTAG, "Finished init thread");
}
runCleanupOperation();
}
};
t.setPriority(Thread.MIN_PRIORITY);
t.start();
}
}

/**
* this could be a long running operation, don't run on the UI thread unless necessary.
* This function prunes the database for old or expired tiles.
*/
public void runCleanupOperation() {
if (db==null) {
if (OpenStreetMapTileProviderConstants.DEBUGMODE) {
Log.d(IMapView.LOGTAG, "Finished init thread, aborted due to null database reference");
}
return;
}

try {
//run the reaper (remove all old expired tiles)
//keep if now is < expiration date
//delete if now is > expiration date
long now = System.currentTimeMillis();
int rows = db.delete(TABLE, "expires < ?", new String[]{System.currentTimeMillis() + ""});
Log.d(IMapView.LOGTAG, "Local storage cache purged " + rows + " expired tiles in " + (System.currentTimeMillis() - now) + "ms, cache size is " + db_file.length() + "bytes");

//VACUUM the database
Log.d(IMapView.LOGTAG, "Local cache is now " + db_file.length() + " max size is " + OpenStreetMapTileProviderConstants.TILE_MAX_CACHE_SIZE_BYTES);
now = System.currentTimeMillis();
//db.execSQL("VACUUM " + DatabaseFileArchive.TABLE + ";");
// Log.d(IMapView.LOGTAG, "VACUUM completed in " + (System.currentTimeMillis()-now) + "ms, cache size is " + db_file.length() + "bytes");
if (db_file.length() > OpenStreetMapTileProviderConstants.TILE_MAX_CACHE_SIZE_BYTES) {
long diff = db_file.length() - OpenStreetMapTileProviderConstants.TILE_MAX_CACHE_SIZE_BYTES ;
long tilesToKill = diff / questimate;
Log.d(IMapView.LOGTAG, "Local cache purging " +tilesToKill + " tiles.");
if (tilesToKill > 0)
try {
db.execSQL("DELETE FROM " + TABLE + " WHERE " + COLUMN_KEY + " in (SELECT " + COLUMN_KEY + " FROM " + TABLE + " ORDER BY expires DESC LIMIT " + tilesToKill + ")");
} catch (Throwable t) {
Log.e(IMapView.LOGTAG, "error purging tiles from the tile cache", t);
}
Log.d(IMapView.LOGTAG, "purge completed in " + (System.currentTimeMillis() - now) + "ms, cache size is " + db_file.length() + " bytes");
}
}catch (Exception ex){
if (OpenStreetMapTileProviderConstants.DEBUGMODE) {
Log.d(IMapView.LOGTAG, "SqliteTileWriter init thread crash, db is probably not available",ex);
}
}

if (OpenStreetMapTileProviderConstants.DEBUGMODE) {
Log.d(IMapView.LOGTAG, "Finished init thread");
}
}

@Override
public boolean saveFile(ITileSource pTileSourceInfo, MapTile pTile, InputStream pStream) {
if (db == null) {
Expand Down Expand Up @@ -132,9 +147,15 @@ public boolean saveFile(ITileSource pTileSourceInfo, MapTile pTile, InputStream
//this shouldn't happen, but just in case
if (pTile.getExpires() != null)
cv.put("expires", pTile.getExpires().getTime());
db.delete(TABLE, DatabaseFileArchive.COLUMN_KEY+"=? and "+DatabaseFileArchive.COLUMN_PROVIDER+"=?", new String[]{index+"",pTileSourceInfo.name()});
db.delete(TABLE, DatabaseFileArchive.COLUMN_KEY + "=? and " + DatabaseFileArchive.COLUMN_PROVIDER + "=?", new String[]{index + "", pTileSourceInfo.name()});
db.insert(TABLE, null, cv);
Log.d(IMapView.LOGTAG, "tile inserted " + pTileSourceInfo.name() + pTile.toString());
Log.d(IMapView.LOGTAG, "tile inserted " + pTileSourceInfo.name() + pTile.toString());
if (db_file.length() > OpenStreetMapTileProviderConstants.TILE_TRIM_CACHE_SIZE_BYTES){
runCleanupOperation();
}
} catch (SQLiteFullException ex) {
//the drive is full! trigger the clean up operation
runCleanupOperation();
} catch (Throwable ex) {
//note, although we check for db null state at the beginning of this method, it's possible for the
//db to be closed during the execution of this method
Expand Down

0 comments on commit bcd59b8

Please sign in to comment.