diff --git a/app/build.gradle b/app/build.gradle index bbdd3870b3..506669f815 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -18,8 +18,8 @@ dependencies { implementation 'com.jakewharton.timber:timber:4.6.0' implementation 'info.debatty:java-string-similarity:0.24' implementation 'com.borjabravo:readmoretextview:2.1.0' - implementation 'com.android.support.constraint:constraint-layout:1.1.3' + implementation "com.android.support:exifinterface:27.1.1" implementation('com.mapbox.mapboxsdk:mapbox-android-sdk:5.5.0@aar') { transitive = true } diff --git a/app/src/androidTest/java/fr/free/nrw/commons/SettingsActivityTest.java b/app/src/androidTest/java/fr/free/nrw/commons/SettingsActivityTest.java index 80caf0010b..ebdc84b152 100644 --- a/app/src/androidTest/java/fr/free/nrw/commons/SettingsActivityTest.java +++ b/app/src/androidTest/java/fr/free/nrw/commons/SettingsActivityTest.java @@ -17,6 +17,7 @@ import org.junit.runner.RunWith; import java.util.Map; +import java.util.Set; import fr.free.nrw.commons.settings.SettingsActivity; @@ -51,6 +52,8 @@ protected void afterActivityFinished() { editor.putBoolean(key, (Boolean)val); } else if (val instanceof Integer) { editor.putInt(key, (Integer)val); + } else if (val instanceof Set){ + editor.putStringSet(key, (Set)val); } else { throw new RuntimeException("type not implemented: " + entry); } diff --git a/app/src/main/java/fr/free/nrw/commons/Utils.java b/app/src/main/java/fr/free/nrw/commons/Utils.java index 5f829abfce..0ef72de3e3 100644 --- a/app/src/main/java/fr/free/nrw/commons/Utils.java +++ b/app/src/main/java/fr/free/nrw/commons/Utils.java @@ -19,7 +19,10 @@ import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.util.Arrays; +import java.util.HashSet; import java.util.Locale; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -224,4 +227,26 @@ public static Bitmap getScreenShot(View view) { return bitmap; } + + public static Set toSet(T... params){ + return new HashSet<>(Arrays.asList(params)); + } + + /** + * Ensures that an object reference passed as a parameter to the calling method is not null. + * + * @param reference an object reference + * @param errorMessage the exception message to use if the check fails; will be converted to a + * string using {@link String#valueOf(Object)} + * @return the non-null reference that was validated + * @throws NullPointerException if {@code reference} is null + */ + public static T checkNotNull(T reference, Object errorMessage) { + if (reference == null) { + throw new NullPointerException(String.valueOf(errorMessage)); + } + return reference; + } + + } diff --git a/app/src/main/java/fr/free/nrw/commons/ui/LongTitlePreferences/LongTitleMultiSelectListPreference.java b/app/src/main/java/fr/free/nrw/commons/ui/LongTitlePreferences/LongTitleMultiSelectListPreference.java new file mode 100644 index 0000000000..07bec5223b --- /dev/null +++ b/app/src/main/java/fr/free/nrw/commons/ui/LongTitlePreferences/LongTitleMultiSelectListPreference.java @@ -0,0 +1,30 @@ +package fr.free.nrw.commons.ui.LongTitlePreferences; + +import android.content.Context; +import android.preference.MultiSelectListPreference; +import android.util.AttributeSet; +import android.view.View; +import android.widget.TextView; + +/** + * Created by Ilgaz Er on 7/31/2018. + */ +public class LongTitleMultiSelectListPreference extends MultiSelectListPreference { + public LongTitleMultiSelectListPreference(Context context, AttributeSet attrs) { + super(context, attrs); + } + + public LongTitleMultiSelectListPreference(Context context) { + super(context); + } + + @Override + protected void onBindView(View view) { + super.onBindView(view); + + TextView title = view.findViewById(android.R.id.title); + if (title != null) { + title.setSingleLine(false); + } + } +} diff --git a/app/src/main/java/fr/free/nrw/commons/upload/FileMetadataUtils.java b/app/src/main/java/fr/free/nrw/commons/upload/FileMetadataUtils.java new file mode 100644 index 0000000000..3fbe2a9309 --- /dev/null +++ b/app/src/main/java/fr/free/nrw/commons/upload/FileMetadataUtils.java @@ -0,0 +1,102 @@ +package fr.free.nrw.commons.upload; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +import io.reactivex.Observable; +import timber.log.Timber; + +import static android.support.media.ExifInterface.*; +import static fr.free.nrw.commons.Utils.checkNotNull; + +public class FileMetadataUtils { + + + public static Observable getTagsFromPref(String pref) { + Timber.d("Retuning tags for pref:" + pref); + switch (pref) { + case "Author": + return Observable.fromArray(TAG_ARTIST, TAG_CAMARA_OWNER_NAME); + case "Copyright": + return Observable.fromArray(TAG_COPYRIGHT); + case "Camera Model": + return Observable.fromArray(TAG_MAKE, TAG_MODEL); + case "Lens Model": + return Observable.fromArray(TAG_LENS_MAKE, TAG_LENS_MODEL, TAG_LENS_SPECIFICATION); + case "Serial Numbers": + return Observable.fromArray(TAG_BODY_SERIAL_NUMBER, TAG_LENS_SERIAL_NUMBER); + case "Software": + return Observable.fromArray(TAG_SOFTWARE); + default: + return null; + } + } + + + /** + * Removes all XMP data from the input file and writes the rest of the image to a new file. + * + * This works by black magic. Pleae read the JPEG section of the XMP Spesification Part 3 before making changes. + * https://wwwimages2.adobe.com/content/dam/acom/en/devnet/xmp/pdfs/XMP%20SDK%20Release%20cc-2016-08/XMPSpecificationPart3.pdf + * + * @param inputPath the path of the input file + * @param outputPath the path of the new file + */ + public static void removeXmpAndWriteToFile(String inputPath, String outputPath) { + try (BufferedInputStream is = new BufferedInputStream(new FileInputStream(inputPath)); + BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(outputPath))) { + int next = 0; + while (next != -1) { + next = is.read(); + //Detect first byte of FF E1, the code of APP1 marker + if (next == 0xFF) { + next = is.read(); + if (next == 0xE1) { + Timber.i("Found FF E1"); + //2 bytes that contain the length of the APP1 section. + byte Lp1 = (byte) is.read(); + byte Lp2 = (byte) is.read(); + Timber.i(Integer.toHexString(Lp1)); + Timber.i(Integer.toHexString(Lp2)); + //The identifier of the APP1 section, we find out if this section contains XMP data or not. + byte[] namespace = new byte[28]; + if (is.read(namespace, 0, 28) != 28) + throw new IOException("Wrong amount of bytes read."); + Timber.i(new String(namespace, "UTF-8")); + if (new String(namespace, "UTF-8").equals("http://ns.adobe.com/xap/1.0/")) { + Timber.i("Found XMP marker"); + while (next != 0xFF) { + if (next == -1) + throw new IOException("Unexpected end of file."); + next = is.read(); + } + //FF means the start of the next marker. + // This means the XMP section is finished and that we should resume copying. + outputStream.write(0xFF); + } else { + //Write everything back to the output file as we want to leave non-XMP APP1 sections as-is. + Timber.i("Not XMP marker"); + outputStream.write(0xFF); + outputStream.write(0xE1); + outputStream.write(Lp1); + outputStream.write(Lp2); + outputStream.write(namespace); + } + } else { + outputStream.write(0xFF); + outputStream.write(next); + } + } else { + outputStream.write(next); + } + } + } catch (IOException e) { + Timber.e(e); + } + } + +} diff --git a/app/src/main/java/fr/free/nrw/commons/upload/FileProcessor.java b/app/src/main/java/fr/free/nrw/commons/upload/FileProcessor.java index b29d686f5b..00e920fb71 100644 --- a/app/src/main/java/fr/free/nrw/commons/upload/FileProcessor.java +++ b/app/src/main/java/fr/free/nrw/commons/upload/FileProcessor.java @@ -1,10 +1,12 @@ package fr.free.nrw.commons.upload; + import android.annotation.SuppressLint; import android.app.Activity; import android.content.ContentResolver; import android.content.Context; import android.content.SharedPreferences; +import android.support.media.ExifInterface; import android.net.Uri; import android.os.Build; import android.os.Bundle; @@ -16,15 +18,21 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.lang.ref.WeakReference; +import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Set; import javax.inject.Inject; import javax.inject.Named; +import fr.free.nrw.commons.R; +import fr.free.nrw.commons.Utils; import fr.free.nrw.commons.caching.CacheController; import fr.free.nrw.commons.di.ApplicationlessInjection; import fr.free.nrw.commons.mwapi.CategoryApi; +import io.reactivex.Observable; import io.reactivex.schedulers.Schedulers; import timber.log.Timber; @@ -51,47 +59,59 @@ public class FileProcessor implements SimilarImageDialogFragment.onResponse { private String decimalCoords; private boolean haveCheckedForOtherImages = false; private String filePath; - private boolean useExtStorage; + private String fileOrCopyPath = null; + private boolean prefUseExtStorage; private boolean cacheFound; private GPSExtractor tempImageObj; + private Set prefManageEXIFTags; + private double prefLocationAccuracy; + private final boolean prefKeepXmp; FileProcessor(Uri mediaUri, ContentResolver contentResolver, Context context) { this.mediaUri = mediaUri; + Timber.d("mediaUri:" + (this.mediaUri != null ? this.mediaUri.getPath() : "null")); this.contentResolver = contentResolver; this.context = context; ApplicationlessInjection.getInstance(context.getApplicationContext()).getCommonsApplicationComponent().inject(this); - useExtStorage = prefs.getBoolean("useExternalStorage", true); + prefUseExtStorage = prefs.getBoolean("useExternalStorage", true); + prefManageEXIFTags = prefs.getStringSet("manageExifTags", Collections.emptySet()); + prefLocationAccuracy = Double.valueOf(prefs.getString("locationAccuracy", "0")) / 111300; //about 111300 meters in one degree + prefKeepXmp = prefs.getBoolean("keepXmp", false); + Timber.d("prefLocationAccuracy:" + prefLocationAccuracy); } /** * Gets file path from media URI. * In older devices getPath() may fail depending on the source URI, creating and using a copy of the file seems to work instead. + * If cleansing EXIF tags is enabled, it always copies the file. * * @return file path of media */ @Nullable private String getPathOfMediaOrCopy() { + if (fileOrCopyPath != null) + return fileOrCopyPath; filePath = FileUtils.getPath(context, mediaUri); Timber.d("Filepath: " + filePath); - if (filePath == null) { - String copyPath = null; + if (filePath == null || !prefManageEXIFTags.isEmpty()) { try { ParcelFileDescriptor descriptor = contentResolver.openFileDescriptor(mediaUri, "r"); if (descriptor != null) { - if (useExtStorage) { - copyPath = FileUtils.createCopyPath(descriptor); - return copyPath; + if (prefUseExtStorage) { + fileOrCopyPath = FileUtils.createCopyPath(descriptor); + return fileOrCopyPath; } - copyPath = getApplicationContext().getCacheDir().getAbsolutePath() + "/" + new Date().getTime() + ".jpg"; - FileUtils.copy(descriptor.getFileDescriptor(), copyPath); - Timber.d("Filepath (copied): %s", copyPath); - return copyPath; + fileOrCopyPath = getApplicationContext().getCacheDir().getAbsolutePath() + "/" + new Date().getTime() + ".jpg"; + FileUtils.copy(descriptor.getFileDescriptor(), fileOrCopyPath); + Timber.d("Filepath (copied): %s", fileOrCopyPath); + return fileOrCopyPath; } } catch (IOException e) { - Timber.w(e, "Error in file " + copyPath); + Timber.w(e, "Error in file " + fileOrCopyPath); return null; } } + fileOrCopyPath = filePath; return filePath; } @@ -130,13 +150,43 @@ GPSExtractor processFileCoordinates(boolean gpsEnabled) { return imageObj; } + /** + * @return The coordinates with reduced accuracy in "lat|long" format + */ String getDecimalCoords() { return decimalCoords; } /** - * Find other images around the same location that were taken within the last 20 sec + * Reduces the accuracy of the coordinate according to location accuracy preference. + * + * @param input + * @return The coordinate with reduced accuracy. + */ + double anonymizeCoord(double input) { + double intermediate = Math.round(input / prefLocationAccuracy) * prefLocationAccuracy; + return Math.round(intermediate * 100000.0) / 100000.0; //Round to 5th decimal place. + } + + /** + * Reduces the accuracy of file coordinates according to location accuracy preference. * + * @return The coordinates with reduced accuracy in "lat|long" format + */ + String getAnonymizedDecimalCoords() { + Timber.d("Anonymizing coords with setting:" + prefLocationAccuracy); + if (prefLocationAccuracy < 0) + return null; + else if (prefLocationAccuracy == 0) + return decimalCoords; + else { + return String.valueOf(anonymizeCoord(imageObj.getDecLatitude())) + "|" + + String.valueOf(anonymizeCoord(imageObj.getDecLongitude())); + } + } + + /** + * Find other images around the same location that were taken within the last 20 sec */ private void findOtherImages() { Timber.d("filePath" + getPathOfMediaOrCopy()); @@ -236,13 +286,60 @@ boolean isCacheFound() { return cacheFound; } + + /** + * Redacts EXIF data from the file. + * + * @return Uri of the new file. + **/ + @SuppressLint("CheckResult") + public Uri redactEXIFData() { + String newFilePath = getPathOfMediaOrCopy(); + try { + Timber.d("Tags to be redacted:" + Arrays.toString(prefManageEXIFTags.toArray())); + Timber.v("File path:" + getPathOfMediaOrCopy()); + if (getPathOfMediaOrCopy() != null) { + if (!prefKeepXmp) { + newFilePath = context.getCacheDir().getAbsolutePath() + "/" + new Date().getTime() + ".jpg"; + FileMetadataUtils.removeXmpAndWriteToFile(getPathOfMediaOrCopy(), newFilePath); + } + ExifInterface exif = new ExifInterface(newFilePath);//Temporary EXIF interface to redact data. + Set redactTags = Utils.toSet(context.getResources().getStringArray(R.array.pref_exifTag_values)); + Timber.d(redactTags.toString()); + redactTags.removeAll(prefManageEXIFTags); + Observable.fromIterable(redactTags) + .flatMap(FileMetadataUtils::getTagsFromPref) + .forEach(tag -> { + Timber.d("Checking for tag:" + tag); + String oldValue = exif.getAttribute(tag); + if (oldValue != null && !oldValue.isEmpty()) { + Timber.d("Exif tag " + tag + " with value " + oldValue + " redacted."); + exif.setAttribute(tag, null); + } + }); + + if (prefLocationAccuracy < 0) { + Timber.d("Setting EXIF coordinates to 0"); + exif.setLatLong(0d, 0d); + } else if (prefLocationAccuracy != 0) { + exif.setLatLong(anonymizeCoord(imageObj.getDecLatitude()), anonymizeCoord(imageObj.getDecLongitude())); + } + exif.saveAttributes(); + } + } catch (IOException e) { + Timber.w(e); + throw new RuntimeException("EXIF redaction failed."); + } + return Uri.parse("file://" + newFilePath); + } + /** * Calls the async task that detects if image is fuzzy, too dark, etc */ void detectUnwantedPictures() { String imageMediaFilePath = FileUtils.getPath(context, mediaUri); DetectUnwantedPicturesAsync detectUnwantedPicturesAsync - = new DetectUnwantedPicturesAsync(new WeakReference((Activity) context), imageMediaFilePath); + = new DetectUnwantedPicturesAsync(new WeakReference<>((Activity) context), imageMediaFilePath); detectUnwantedPicturesAsync.execute(); } diff --git a/app/src/main/java/fr/free/nrw/commons/upload/MultipleShareActivity.java b/app/src/main/java/fr/free/nrw/commons/upload/MultipleShareActivity.java index a35eb46eec..47bf87daa8 100644 --- a/app/src/main/java/fr/free/nrw/commons/upload/MultipleShareActivity.java +++ b/app/src/main/java/fr/free/nrw/commons/upload/MultipleShareActivity.java @@ -386,16 +386,21 @@ private void prepareMultipleUploadList() { for (int i = 0; i < urisList.size(); i++) { Contribution up = new Contribution(); Uri uri = urisList.get(i); + + FileProcessor fileObj= new FileProcessor(uri, this.getContentResolver(), this); + fileObj.processFileCoordinates(false); + // Use temporarily saved file Uri instead - uri = ContributionUtils.saveFileBeingUploadedTemporarily(this, uri); + uri = ContributionUtils.saveFileBeingUploadedTemporarily(this, fileObj.redactEXIFData()); up.setLocalUri(uri); + up.setTag("mimeType", intent.getType()); up.setTag("sequence", i); up.setSource(Contribution.SOURCE_EXTERNAL); up.setMultiple(true); - String imageGpsCoordinates = extractImageGpsData(uri); + String imageGpsCoordinates=fileObj.getAnonymizedDecimalCoords(); if (imageGpsCoordinates != null) { - Timber.d("GPS data for image found!"); + Timber.d("GPS data will be used for image!"); up.setDecimalCoords(imageGpsCoordinates); } photosList.add(up); diff --git a/app/src/main/java/fr/free/nrw/commons/upload/ShareActivity.java b/app/src/main/java/fr/free/nrw/commons/upload/ShareActivity.java index 3a8e1b4138..7e3b46ef03 100644 --- a/app/src/main/java/fr/free/nrw/commons/upload/ShareActivity.java +++ b/app/src/main/java/fr/free/nrw/commons/upload/ShareActivity.java @@ -130,6 +130,7 @@ public class ShareActivity private String mimeType; private CategorizationFragment categorizationFragment; private Uri mediaUri; + private Uri redactedMediaUri; private Uri contentProviderUri; private Contribution contribution; private GPSExtractor gpsObj; @@ -219,13 +220,15 @@ private void uploadBegins() { Timber.d("Cache the categories found"); } - uploadController.startUpload(title, contentProviderUri, mediaUri, description, mimeType, source, decimalCoords, wikiDataEntityId, c -> { + redactedMediaUri=fileObj.redactEXIFData(); + + uploadController.startUpload(title, contentProviderUri, redactedMediaUri, description, mimeType, source, decimalCoords, wikiDataEntityId, c -> { ShareActivity.this.contribution = c; showPostUpload(); }); isUploadFinalised = true; } - + /** * Starts CategorizationFragment after uploadBegins. */ @@ -478,7 +481,7 @@ private void checkIfFileExists() { Timber.d("File SHA1 is: %s", fileSHA1); ExistingFileAsync fileAsyncTask = - new ExistingFileAsync(new WeakReference(this), fileSHA1, new WeakReference(this), result -> { + new ExistingFileAsync(new WeakReference<>(this), fileSHA1, new WeakReference<>(this), result -> { Timber.d("%s duplicate check: %s", mediaUri.toString(), result); duplicateCheckPassed = (result == DUPLICATE_PROCEED || result == NO_DUPLICATE); if (duplicateCheckPassed) { diff --git a/app/src/main/res/values/arrays.xml b/app/src/main/res/values/arrays.xml index 3cfe500d49..2fff08dfd3 100644 --- a/app/src/main/res/values/arrays.xml +++ b/app/src/main/res/values/arrays.xml @@ -14,4 +14,50 @@ @string/license_pref_cc_by_sa_3_0 @string/license_pref_cc_by_sa_4_0 + + + @string/exif_tag_name_author + @string/exif_tag_name_copyright + @string/exif_tag_name_model + @string/exif_tag_name_lens_model + @string/exif_tag_name_serial + @string/exif_tag_name_software + + + + @string/exif_tag_author + @string/exif_tag_copyright + @string/exif_tag_model + @string/exif_tag_lens_model + @string/exif_tag_serial + @string/exif_tag_software + + + + @string/exif_tag_author + @string/exif_tag_copyright + @string/exif_tag_model + @string/exif_tag_lens_model + @string/exif_tag_software + + + + + @string/location_accuracy_pref_remove + @string/location_accuracy_pref_20km + @string/location_accuracy_pref_10km + @string/location_accuracy_pref_5km + @string/location_accuracy_pref_1km + @string/location_accuracy_pref_100m + @string/location_accuracy_pref_do_nothing + + + -1 + 20000 + 10000 + 5000 + 1000 + 100 + 0 + \ No newline at end of file diff --git a/app/src/main/res/values/keys.xml b/app/src/main/res/values/keys.xml index a2bb6860d2..c47b90d591 100644 --- a/app/src/main/res/values/keys.xml +++ b/app/src/main/res/values/keys.xml @@ -5,4 +5,11 @@ CC BY-SA 3.0 CC BY 4.0 CC BY-SA 4.0 + + Author + Copyright + Camera Model + Lens Model + Serial Numbers + Software \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index a0571ee42d..5a7c780af7 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -309,7 +309,26 @@ Failed to update corresponding Wikidata entity! Set as wallpaper Wallpaper set successfully! - Quiz + + Manage EXIF Tags + Select which EXIF tags to include in uploads. + Set Location Accuracy + Remove all location data. + Reduce accuracy to 20km + Reduce accuracy to 10km + Reduce accuracy to 5km + Reduce accuracy to 1km + Reduce accuracy to 100m + Do nothing + + Author + Copyright + Camera Model + Lens Model + Serial Numbers + Software + +Quiz Is this picture OK to upload? Question Result @@ -351,7 +370,8 @@ The number of images you have uploaded to Commons, via any upload software The percentage of images you have uploaded to Commons that were not deleted The number of images you have uploaded to Commons that were used in Wikimedia articles - + Keep XMP Metadata + XMP is another form of image metadata storage, but it is currently not possible to remove individual XMP tags from the image. You are encouraged to disable this option if anonymity is a concern. Error occurred! Commons Notification Storage Permission diff --git a/app/src/main/res/xml/preferences.xml b/app/src/main/res/xml/preferences.xml index 37b6a8b01d..a72624dea0 100644 --- a/app/src/main/res/xml/preferences.xml +++ b/app/src/main/res/xml/preferences.xml @@ -1,6 +1,5 @@ - + @@ -8,8 +7,8 @@ + android:key="theme" + android:summary="@string/preference_theme_summary" /> @@ -22,25 +21,51 @@ android:entries="@array/pref_defaultLicense_entries" android:entryValues="@array/pref_defaultLicense_values" android:defaultValue="@string/license_pref_cc_by_sa_4_0" /> - + + android:key="useExternalStorage" + android:summary="@string/use_external_storage_summary"/> + android:maxLength="3" + android:title="@string/set_limit" /> - + + + + + + + @@ -52,8 +77,8 @@ + android:summary="@string/send_log_file_description" + android:title="@string/send_log_file" /> \ No newline at end of file