From 69d2f95b9b1f0eb41168330707cf9f47f2004e5e Mon Sep 17 00:00:00 2001 From: Thomas More Date: Wed, 3 Apr 2024 13:43:02 -0400 Subject: [PATCH] Implement deduplication of call log entries Closes: #176 --- README.md | 8 +- .../github/tmo1/sms_ie/ImportExportCallLog.kt | 87 ++++++++++--------- .../tmo1/sms_ie/ImportExportMessages.kt | 2 +- app/src/main/res/xml/root_preferences.xml | 4 +- .../metadata/android/en-US/changelogs/106.txt | 2 + 5 files changed, 56 insertions(+), 47 deletions(-) create mode 100644 fastlane/metadata/android/en-US/changelogs/106.txt diff --git a/README.md b/README.md index c928bed..6f350f8 100644 --- a/README.md +++ b/README.md @@ -84,17 +84,17 @@ Additionally, some MMS part metadata apparently contain a `sub_id` field as well ### Deduplication -SMS Import / Export can attempt to deduplicate messages upon import. If this feature is enabled in the app's settings, the app will check all new messages against the existing message database (including those messages already inserted earlier in the import process) and ignore those it considers to be duplicates of ones already present. This feature is currently considered experimental. +SMS Import / Export can attempt to deduplicate messages and call log entries upon import. If this feature is enabled in the app's settings, the app will check all new messages and call log entries against all existing messages and call log entries in the respective databases (including those messages and call log entries already inserted earlier in the import process) and ignore those it considers to be duplicates of ones already in the databases. This feature is currently considered experimental; it has not been extensively tested, and may yield both false positives and false negatives, and is accordingly not enabled by default. Deduplication of contacts is not currently implemented. -If this feature is not enabled, no deduplication is done. For example, if messages are exported and then immediately reimported, the device will then contain two copies of every message. To avoid this, the device can be wiped of all messages before importing by using the `Wipe Messages` button. +If this feature is not enabled, no deduplication is done. For example, if messages are exported and then immediately reimported, the device will then contain two copies of every message. To avoid this, the device can be wiped of all messages before importing by using the "Wipe Messages" button. The call log can be cleared via the standard phone app: select "Call history" from the app's ellipsis menu, then select "Clear call history" from the call history ellipsis menu. -SMS Import / Export cannot directly deduplicate messages already present in the Android database, but it should be possible to use the app to perform such deduplication by first exporting messages, then wiping messages, and finally re-importing the exported messages. +SMS Import / Export cannot directly deduplicate messages or call log entries already present in the Android databases, but it should be possible to use the app to perform such deduplication by first exporting messages / call log, then wiping messages / clearing the call history, and finally re-importing the exported messages / call log. #### Implementation Message deduplication is tricky, since on the one hand, unlike email messages, SMS and MMS messages do not generally have a unique [`Message-ID`](https://en.wikipedia.org/wiki/Message-ID), while on the other hand, some message metadata (e.g., [`THREAD_ID`](https://developer.android.com/reference/android/provider/Telephony.TextBasedSmsColumns#THREAD_ID)) does not remain constant when messages are moved around, and some metadata is not present for all messages. SMS Import / Export therefore tries to identify duplicate messages by comparing carefully chosen message data and metadata fields and concludes that two messages are identical if the compared fields are. Currently, SMS messages are assumed to be identical if they have identical [`ADDRESS`](https://developer.android.com/reference/android/provider/Telephony.TextBasedSmsColumns#ADDRESS), [`TYPE`](https://developer.android.com/reference/android/provider/Telephony.TextBasedSmsColumns#TYPE), [`DATE`](https://developer.android.com/reference/android/provider/Telephony.TextBasedSmsColumns#DATE), and [`BODY`](https://developer.android.com/reference/android/provider/Telephony.TextBasedSmsColumns#BODY) fields, and MMS messages are assumed to be identical if they have identical [`DATE`](https://developer.android.com/reference/android/provider/Telephony.BaseMmsColumns#DATE) and [`MESSAGE_BOX`](https://developer.android.com/reference/android/provider/Telephony.BaseMmsColumns#MESSAGE_BOX) fields, plus identical [`MESSAGE_ID`](https://developer.android.com/reference/android/provider/Telephony.BaseMmsColumns#MESSAGE_ID) fields if that field is present in the new message, or identical [`CONTENT_LOCATION`](https://developer.android.com/reference/android/provider/Telephony.BaseMmsColumns#CONTENT_LOCATION) fields if that field is present in the new message and `MESSAGE_ID` is not. -This functionality has not been extensively tested, and may yield both false positives and false negatives. +Call log deduplication works similarly but is simpler: call log entries are assumed to be identical if they have identical [`NUMBER`](https://developer.android.com/reference/android/provider/CallLog.Calls#NUMBER), [`TYPE`](https://developer.android.com/reference/android/provider/CallLog.Calls#TYPE), and [`DATE`](https://developer.android.com/reference/android/provider/CallLog.Calls#DATE) fields. ### Scheduled Export diff --git a/app/src/main/java/com/github/tmo1/sms_ie/ImportExportCallLog.kt b/app/src/main/java/com/github/tmo1/sms_ie/ImportExportCallLog.kt index a92db08..07635b4 100644 --- a/app/src/main/java/com/github/tmo1/sms_ie/ImportExportCallLog.kt +++ b/app/src/main/java/com/github/tmo1/sms_ie/ImportExportCallLog.kt @@ -41,10 +41,7 @@ import java.io.InputStreamReader import java.io.OutputStreamWriter suspend fun exportCallLog( - appContext: Context, - file: Uri, - progressBar: ProgressBar?, - statusReportText: TextView? + appContext: Context, file: Uri, progressBar: ProgressBar?, statusReportText: TextView? ): MessageTotal { //val prefs = PreferenceManager.getDefaultSharedPreferences(appContext) return withContext(Dispatchers.IO) { @@ -56,11 +53,7 @@ suspend fun exportCallLog( jsonWriter.setIndent(" ") jsonWriter.beginArray() totals.sms = callLogToJSON( - appContext, - jsonWriter, - displayNames, - progressBar, - statusReportText + appContext, jsonWriter, displayNames, progressBar, statusReportText ) jsonWriter.endArray() } @@ -78,14 +71,9 @@ private suspend fun callLogToJSON( ): Int { val prefs = PreferenceManager.getDefaultSharedPreferences(appContext) var total = 0 - val callCursor = - appContext.contentResolver.query( - Uri.parse("content://call_log/calls"), - null, - null, - null, - null - ) + val callCursor = appContext.contentResolver.query( + Uri.parse("content://call_log/calls"), null, null, null, null + ) callCursor?.use { if (it.moveToFirst()) { val totalCalls = it.count @@ -103,8 +91,7 @@ private suspend fun callLogToJSON( // This value is typically filled in by the dialer app for the caching purpose, so it's not guaranteed to be present, and may not be current if the contact information associated with this number has changed." val address = it.getString(addressIndex) if (address != null) { - val displayName = - lookupDisplayName(appContext, displayNames, address) + val displayName = lookupDisplayName(appContext, displayNames, address) if (displayName != null) jsonWriter.name("display_name").value(displayName) } jsonWriter.endObject() @@ -123,16 +110,14 @@ private suspend fun callLogToJSON( } suspend fun importCallLog( - appContext: Context, - uri: Uri, - progressBar: ProgressBar, - statusReportText: TextView + appContext: Context, uri: Uri, progressBar: ProgressBar, statusReportText: TextView ): Int { + val prefs = PreferenceManager.getDefaultSharedPreferences(appContext) + val deduplication = prefs.getBoolean("deduplication", false) return withContext(Dispatchers.IO) { val callLogColumns = mutableSetOf() val callLogCursor = appContext.contentResolver.query( - CallLog.Calls.CONTENT_URI, - null, null, null, null + CallLog.Calls.CONTENT_URI, null, null, null, null ) callLogCursor?.use { callLogColumns.addAll(it.columnNames) } var callLogCount = 0 @@ -144,25 +129,53 @@ suspend fun importCallLog( val callLogMetadata = ContentValues() try { jsonReader.beginArray() - while (jsonReader.hasNext()) { + JSONReader@ while (jsonReader.hasNext()) { jsonReader.beginObject() callLogMetadata.clear() while (jsonReader.hasNext()) { val name = jsonReader.nextName() val value = jsonReader.nextString() if ((callLogColumns.contains(name)) and (name !in setOf( - BaseColumns._ID, - BaseColumns._COUNT + BaseColumns._ID, BaseColumns._COUNT )) ) { callLogMetadata.put(name, value) } } + jsonReader.endObject() + if (deduplication) { + val callDuplicatesCursor = appContext.contentResolver.query( + CallLog.Calls.CONTENT_URI, + arrayOf(CallLog.Calls._ID), + "${CallLog.Calls.NUMBER}=? AND ${CallLog.Calls.TYPE}=? AND ${CallLog.Calls.DATE}=?", + arrayOf( + callLogMetadata.getAsString(CallLog.Calls.NUMBER), + callLogMetadata.getAsString(CallLog.Calls.TYPE), + callLogMetadata.getAsString(CallLog.Calls.DATE) + + ), + null + ) + val isDuplicate = callDuplicatesCursor?.use { _ -> + if (callDuplicatesCursor.moveToFirst()) { + Log.d(LOG_TAG, "Duplicate call - skipping") + true + } else { + false + } + } + if (isDuplicate == true) { + continue@JSONReader + } + } var insertUri: Uri? = null - if (callLogMetadata.keySet().contains(CallLog.Calls.NUMBER) && callLogMetadata.getAsString(CallLog.Calls.TYPE) != "4") { + if (callLogMetadata.keySet() + .contains(CallLog.Calls.NUMBER) && callLogMetadata.getAsString( + CallLog.Calls.TYPE + ) != "4" + ) { insertUri = appContext.contentResolver.insert( - CallLog.Calls.CONTENT_URI, - callLogMetadata + CallLog.Calls.CONTENT_URI, callLogMetadata ) } if (insertUri == null) { @@ -170,22 +183,16 @@ suspend fun importCallLog( } else { callLogCount++ setStatusText( - statusReportText, - appContext.getString( - R.string.call_log_import_progress, - callLogCount + statusReportText, appContext.getString( + R.string.call_log_import_progress, callLogCount ) ) } - jsonReader.endObject() } jsonReader.endArray() } catch (e: Exception) { displayError( - appContext, - e, - "Error importing call log", - "Error parsing JSON" + appContext, e, "Error importing call log", "Error parsing JSON" ) } } diff --git a/app/src/main/java/com/github/tmo1/sms_ie/ImportExportMessages.kt b/app/src/main/java/com/github/tmo1/sms_ie/ImportExportMessages.kt index 5e6ca16..7b1e17a 100644 --- a/app/src/main/java/com/github/tmo1/sms_ie/ImportExportMessages.kt +++ b/app/src/main/java/com/github/tmo1/sms_ie/ImportExportMessages.kt @@ -291,7 +291,7 @@ suspend fun importMessages( appContext: Context, uri: Uri, progressBar: ProgressBar?, statusReportText: TextView? ): MessageTotal { val prefs = PreferenceManager.getDefaultSharedPreferences(appContext) - val deduplication = prefs.getBoolean("message_deduplication", false) + val deduplication = prefs.getBoolean("deduplication", false) return withContext(Dispatchers.IO) { val totals = MessageTotal() // get column names of local SMS, MMS, and MMS part tables diff --git a/app/src/main/res/xml/root_preferences.xml b/app/src/main/res/xml/root_preferences.xml index 1cdcf26..1ba0896 100644 --- a/app/src/main/res/xml/root_preferences.xml +++ b/app/src/main/res/xml/root_preferences.xml @@ -96,8 +96,8 @@ diff --git a/fastlane/metadata/android/en-US/changelogs/106.txt b/fastlane/metadata/android/en-US/changelogs/106.txt new file mode 100644 index 0000000..16edca6 --- /dev/null +++ b/fastlane/metadata/android/en-US/changelogs/106.txt @@ -0,0 +1,2 @@ + - Implement deduplication of call log entries (issue #176). +