Skip to content

Commit

Permalink
Move logging into a database.
Browse files Browse the repository at this point in the history
  • Loading branch information
greyson-signal committed Jul 21, 2021
1 parent 0b85852 commit 7419da7
Show file tree
Hide file tree
Showing 27 changed files with 724 additions and 443 deletions.
Expand Up @@ -27,14 +27,11 @@

import com.google.android.gms.security.ProviderInstaller;

import net.sqlcipher.database.SQLiteDatabase;

import org.conscrypt.Conscrypt;
import org.signal.aesgcmprovider.AesGcmProvider;
import org.signal.core.util.concurrent.SignalExecutors;
import org.signal.core.util.logging.AndroidLogger;
import org.signal.core.util.logging.Log;
import org.signal.core.util.logging.PersistentLogger;
import org.signal.core.util.tracing.Tracer;
import org.signal.glide.SignalGlideCodecs;
import org.signal.ringrtc.CallManager;
Expand All @@ -56,7 +53,7 @@
import org.thoughtcrime.securesms.jobs.RetrieveProfileJob;
import org.thoughtcrime.securesms.keyvalue.SignalStore;
import org.thoughtcrime.securesms.logging.CustomSignalProtocolLogger;
import org.thoughtcrime.securesms.logging.LogSecretProvider;
import org.thoughtcrime.securesms.logging.PersistentLogger;
import org.thoughtcrime.securesms.messageprocessingalarm.MessageProcessReceiver;
import org.thoughtcrime.securesms.migrations.ApplicationMigrations;
import org.thoughtcrime.securesms.notifications.NotificationChannels;
Expand All @@ -74,7 +71,6 @@
import org.thoughtcrime.securesms.storage.StorageSyncHelper;
import org.thoughtcrime.securesms.util.AppForegroundObserver;
import org.thoughtcrime.securesms.util.AppStartup;
import org.thoughtcrime.securesms.util.ByteUnit;
import org.thoughtcrime.securesms.util.DynamicTheme;
import org.thoughtcrime.securesms.util.FeatureFlags;
import org.thoughtcrime.securesms.util.SignalUncaughtExceptionHandler;
Expand Down Expand Up @@ -248,7 +244,7 @@ private void initializeSecurityProvider() {
}

private void initializeLogging() {
persistentLogger = new PersistentLogger(this, LogSecretProvider.getOrCreateAttachmentSecret(this), BuildConfig.VERSION_NAME, FeatureFlags.internalUser() ? 15 : 7, ByteUnit.KILOBYTES.toBytes(300));
persistentLogger = new PersistentLogger(this, TimeUnit.DAYS.toMillis(2));
org.signal.core.util.logging.Log.initialize(FeatureFlags::internalUser, new AndroidLogger(), persistentLogger);

SignalProtocolLoggerProvider.setProvider(new CustomSignalProtocolLogger());
Expand Down
Expand Up @@ -61,7 +61,7 @@ public class KeyValueDatabase extends SQLiteOpenHelper implements SignalDatabase
return instance;
}

public KeyValueDatabase(@NonNull Application application, @NonNull DatabaseSecret databaseSecret) {
private KeyValueDatabase(@NonNull Application application, @NonNull DatabaseSecret databaseSecret) {
super(application, DATABASE_NAME, null, DATABASE_VERSION, new SqlCipherDatabaseHook(), new SqlCipherErrorHandler(DATABASE_NAME));

this.application = application;
Expand Down
166 changes: 166 additions & 0 deletions app/src/main/java/org/thoughtcrime/securesms/database/LogDatabase.kt
@@ -0,0 +1,166 @@
package org.thoughtcrime.securesms.database

import android.annotation.SuppressLint
import android.app.Application
import android.content.ContentValues
import android.database.Cursor
import net.sqlcipher.database.SQLiteDatabase
import net.sqlcipher.database.SQLiteOpenHelper
import org.signal.core.util.logging.Log
import org.thoughtcrime.securesms.crypto.DatabaseSecret
import org.thoughtcrime.securesms.crypto.DatabaseSecretProvider
import org.thoughtcrime.securesms.database.model.LogEntry
import org.thoughtcrime.securesms.util.CursorUtil
import org.thoughtcrime.securesms.util.SqlUtil
import java.io.Closeable

/**
* Stores logs.
*
* Logs are very performance-critical, particularly inserts and deleting old entries.
*
* This is it's own separate physical database, so it cannot do joins or queries with any other
* tables.
*/
class LogDatabase private constructor(
application: Application,
private val databaseSecret: DatabaseSecret
) : SQLiteOpenHelper(
application,
DATABASE_NAME,
null,
DATABASE_VERSION,
SqlCipherDatabaseHook(),
SqlCipherErrorHandler(DATABASE_NAME)
),
SignalDatabase {

companion object {
private val TAG = Log.tag(LogDatabase::class.java)

private const val DATABASE_VERSION = 1
private const val DATABASE_NAME = "signal-logs.db"

private const val TABLE_NAME = "log"
private const val ID = "_id"
private const val CREATED_AT = "created_at"
private const val EXPIRES_AT = "expires_at"
private const val BODY = "body"
private const val SIZE = "size"

private val CREATE_TABLE = """
CREATE TABLE $TABLE_NAME (
$ID INTEGER PRIMARY KEY,
$CREATED_AT INTEGER,
$EXPIRES_AT INTEGER,
$BODY TEXT,
$SIZE INTEGER
)
""".trimIndent()

private val CREATE_INDEXES = arrayOf(
"CREATE INDEX log_expires_at_index ON $TABLE_NAME ($EXPIRES_AT)"
)

@SuppressLint("StaticFieldLeak") // We hold an Application context, not a view context
@Volatile
private var instance: LogDatabase? = null

@JvmStatic
fun getInstance(context: Application): LogDatabase {
if (instance == null) {
synchronized(LogDatabase::class.java) {
if (instance == null) {
instance = LogDatabase(context, DatabaseSecretProvider.getOrCreateDatabaseSecret(context))
}
}
}
return instance!!
}
}

override fun onCreate(db: SQLiteDatabase) {
Log.i(TAG, "onCreate()")
db.execSQL(CREATE_TABLE)
CREATE_INDEXES.forEach { db.execSQL(it) }
}

override fun onUpgrade(db: SQLiteDatabase, oldVersion: Int, newVersion: Int) {
Log.i(TAG, "onUpgrade($oldVersion, $newVersion)")
}

override fun getSqlCipherDatabase(): SQLiteDatabase {
return writableDatabase
}

fun insert(logs: List<LogEntry>, currentTime: Long) {
val db = writableDatabase

db.beginTransaction()
try {
logs.forEach { log ->
db.insert(TABLE_NAME, null, buildValues(log))
}
db.delete(TABLE_NAME, "$EXPIRES_AT < ?", SqlUtil.buildArgs(currentTime))
db.setTransactionSuccessful()
} finally {
db.endTransaction()
}
}

fun getAllBeforeTime(time: Long): Reader {
return Reader(readableDatabase.query(TABLE_NAME, arrayOf(BODY), "$CREATED_AT < ?", SqlUtil.buildArgs(time), null, null, null))
}

fun getRangeBeforeTime(start: Int, length: Int, time: Long): List<String> {
val lines = mutableListOf<String>()

readableDatabase.query(TABLE_NAME, arrayOf(BODY), "$CREATED_AT < ?", SqlUtil.buildArgs(time), null, null, null, "$start,$length").use { cursor ->
while (cursor.moveToNext()) {
lines.add(CursorUtil.requireString(cursor, BODY))
}
}

return lines
}

fun getLogCountBeforeTime(time: Long): Int {
readableDatabase.query(TABLE_NAME, arrayOf("COUNT(*)"), "$CREATED_AT < ?", SqlUtil.buildArgs(time), null, null, null).use { cursor ->
return if (cursor.moveToFirst()) {
cursor.getInt(0)
} else {
0
}
}
}

private fun buildValues(log: LogEntry): ContentValues {
return ContentValues().apply {
put(CREATED_AT, log.createdAt)
put(EXPIRES_AT, log.createdAt + log.lifespan)
put(BODY, log.body)
put(SIZE, log.body.length)
}
}

private val readableDatabase: SQLiteDatabase
get() = getReadableDatabase(databaseSecret.asString())

private val writableDatabase: SQLiteDatabase
get() = getWritableDatabase(databaseSecret.asString())

class Reader(private val cursor: Cursor) : Iterator<String>, Closeable {
override fun hasNext(): Boolean {
return !cursor.isLast
}

override fun next(): String {
cursor.moveToNext()
return CursorUtil.requireString(cursor, BODY)
}

override fun close() {
cursor.close()
}
}
}
@@ -0,0 +1,7 @@
package org.thoughtcrime.securesms.database.model

data class LogEntry(
val createdAt: Long,
val lifespan: Long,
val body: String
)
Expand Up @@ -19,29 +19,19 @@ public class HelpViewModel extends ViewModel {

private static final int MINIMUM_PROBLEM_CHARS = 10;

private MutableLiveData<Boolean> problemMeetsLengthRequirements = new MutableLiveData<>();
private MutableLiveData<Boolean> hasLines = new MutableLiveData<>(false);
private MutableLiveData<Integer> categoryIndex = new MutableLiveData<>(0);
private LiveData<Boolean> isFormValid = Transformations.map(new LiveDataPair<>(problemMeetsLengthRequirements, hasLines), this::transformValidationData);
private final MutableLiveData<Boolean> problemMeetsLengthRequirements;
private final MutableLiveData<Integer> categoryIndex;
private final LiveData<Boolean> isFormValid;

private final SubmitDebugLogRepository submitDebugLogRepository;

private List<LogLine> logLines;

public HelpViewModel() {
submitDebugLogRepository = new SubmitDebugLogRepository();

submitDebugLogRepository.getLogLines(lines -> {
logLines = lines;
hasLines.postValue(true);
});
submitDebugLogRepository = new SubmitDebugLogRepository();
problemMeetsLengthRequirements = new MutableLiveData<>();
categoryIndex = new MutableLiveData<>(0);

LiveData<Boolean> firstValid = LiveDataUtil.combineLatest(problemMeetsLengthRequirements, hasLines, (validLength, validLines) -> {
return validLength == Boolean.TRUE && validLines == Boolean.TRUE;
});

isFormValid = LiveDataUtil.combineLatest(firstValid, categoryIndex, (valid, index) -> {
return valid == Boolean.TRUE && index > 0;
isFormValid = LiveDataUtil.combineLatest(problemMeetsLengthRequirements, categoryIndex, (meetsLengthRequirements, index) -> {
return meetsLengthRequirements == Boolean.TRUE && index > 0;
});
}

Expand All @@ -65,18 +55,14 @@ LiveData<SubmitResult> onSubmitClicked(boolean includeDebugLogs) {
MutableLiveData<SubmitResult> resultLiveData = new MutableLiveData<>();

if (includeDebugLogs) {
submitDebugLogRepository.submitLog(logLines, result -> resultLiveData.postValue(new SubmitResult(result, result.isPresent())));
submitDebugLogRepository.buildAndSubmitLog(result -> resultLiveData.postValue(new SubmitResult(result, result.isPresent())));
} else {
resultLiveData.postValue(new SubmitResult(Optional.absent(), false));
}

return resultLiveData;
}

private boolean transformValidationData(Pair<Boolean, Boolean> validationData) {
return validationData.first() == Boolean.TRUE && validationData.second() == Boolean.TRUE;
}

static class SubmitResult {
private final Optional<String> debugLogUrl;
private final boolean isError;
Expand Down
Expand Up @@ -39,6 +39,7 @@
import org.thoughtcrime.securesms.migrations.BlobStorageLocationMigrationJob;
import org.thoughtcrime.securesms.migrations.CachedAttachmentsMigrationJob;
import org.thoughtcrime.securesms.migrations.DatabaseMigrationJob;
import org.thoughtcrime.securesms.migrations.DeleteDeprecatedLogsMigrationJob;
import org.thoughtcrime.securesms.migrations.DirectoryRefreshMigrationJob;
import org.thoughtcrime.securesms.migrations.KbsEnclaveMigrationJob;
import org.thoughtcrime.securesms.migrations.LegacyMigrationJob;
Expand Down Expand Up @@ -175,6 +176,7 @@ public static Map<String, Job.Factory> getJobFactories(@NonNull Application appl
put(BlobStorageLocationMigrationJob.KEY, new BlobStorageLocationMigrationJob.Factory());
put(CachedAttachmentsMigrationJob.KEY, new CachedAttachmentsMigrationJob.Factory());
put(DatabaseMigrationJob.KEY, new DatabaseMigrationJob.Factory());
put(DeleteDeprecatedLogsMigrationJob.KEY, new DeleteDeprecatedLogsMigrationJob.Factory());
put(DirectoryRefreshMigrationJob.KEY, new DirectoryRefreshMigrationJob.Factory());
put(KbsEnclaveMigrationJob.KEY, new KbsEnclaveMigrationJob.Factory());
put(LegacyMigrationJob.KEY, new LegacyMigrationJob.Factory());
Expand Down

0 comments on commit 7419da7

Please sign in to comment.