TIKA-2861: Extract QuickTime GPS location and item-list metadata in MP4Parser

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4BoxHandler.java

@@ -17,6 +17,14 @@
 package org.apache.tika.parser.mp4;
 
 import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import com.drew.imaging.mp4.Mp4Handler;
 import com.drew.lang.annotations.NotNull;
@@ -26,13 +34,31 @@
 import com.drew.metadata.mp4.Mp4Context;
 import org.xml.sax.SAXException;
 
+import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.parser.mp4.boxes.TikaUserDataBox;
 import org.apache.tika.sax.XHTMLContentHandler;
 
 public class TikaMp4BoxHandler extends Mp4BoxHandler {
 
+    //QTFF "well-known" metadata item value types
+    private static final int QT_TEXT_TYPE = 1;
+    private static final int QT_INT_BE_TYPE = 21;
+    private static final int QT_UINT_BE_TYPE = 22;
+    private static final int QT_FLOAT32_TYPE = 23;
+    private static final int QT_FLOAT64_TYPE = 24;
+
+    //QuickTime stores location as an ISO 6709 string (e.g. +32.4720-084.9952+073.827/)
+    private static final String QT_LOCATION_ISO6709 = "com.apple.quicktime.location.ISO6709";
+    private static final Pattern ISO6709_PATTERN =
+            Pattern.compile("([+-]\\d+(?:\\.\\d+)?)([+-]\\d+(?:\\.\\d+)?)([+-]\\d+(?:\\.\\d+)?)?");
+
     org.apache.tika.metadata.Metadata tikaMetadata;
     final XHTMLContentHandler xhtml;
+
+    //key names for the current 'meta' box, filled from its 'keys' box and consumed
+    //by the following 'ilst' box (e.g. com.apple.quicktime.content.identifier)
+    private final List<String> quickTimeMetadataKeys = new ArrayList<>();
+
     public TikaMp4BoxHandler(Metadata metadata, org.apache.tika.metadata.Metadata tikaMetadata,
                              XHTMLContentHandler xhtml) {
         super(metadata);
@@ -42,7 +68,7 @@ public TikaMp4BoxHandler(Metadata metadata, org.apache.tika.metadata.Metadata ti
 
     @Override
     public boolean shouldAcceptBox(@NotNull String box) {
-        if (box.equals("udta")) {
+        if (box.equals("udta") || box.equals("keys") || box.equals("ilst")) {
             return true;
         }
         return super.shouldAcceptBox(box);
@@ -59,6 +85,12 @@ public Mp4Handler<?> processBox(@NotNull String box, @Nullable byte[] payload,
             throws IOException {
         if (box.equals("udta")) {
             return processUserData(box, payload, context);
+        } else if (box.equals("keys")) {
+            processQuickTimeKeys(payload);
+            return this;
+        } else if (box.equals("ilst")) {
+            processQuickTimeItemList(payload);
+            return this;
         }
 
         return super.processBox(box, payload, size, context);
@@ -76,4 +108,124 @@ private Mp4Handler<?> processUserData(String box, byte[] payload, Mp4Context con
         }
         return this;
     }
+
+    /**
+     * Parses the QuickTime metadata 'keys' box, which maps 1-based indices to key
+     * names such as {@code com.apple.quicktime.content.identifier}. The base MP4
+     * handler descends into the enclosing 'meta' container but skips 'keys'/'ilst',
+     * so this metadata (content identifier, ISO 6709 location, make/model, ...) was
+     * previously dropped for QuickTime .mov (and any .mp4 carrying it).
+     */
+    private void processQuickTimeKeys(@Nullable byte[] payload) {
+        quickTimeMetadataKeys.clear();
+        if (payload == null || payload.length < 8) {
+            return;
+        }
+        //1 byte version + 3 bytes flags, then uint32 entry count
+        int pos = 4;
+        long entryCount = readUInt32(payload, pos);
+        pos += 4;
+        for (long i = 0; i < entryCount && pos + 8 <= payload.length; i++) {
+            long keySize = readUInt32(payload, pos);
+            if (keySize < 8 || pos + keySize > payload.length) {
+                return;
+            }
+            //4 bytes key namespace, then the UTF-8 key name
+            quickTimeMetadataKeys.add(
+                    new String(payload, pos + 8, (int) keySize - 8, StandardCharsets.UTF_8));
+            pos += (int) keySize;
+        }
+    }
+
+    /**
+     * Parses the QuickTime metadata 'ilst' box, whose entries are keyed by the
+     * 1-based index into the preceding 'keys' box. Each entry holds a 'data' box
+     * with the value. UTF-8 text and the numeric "well-known" value types are emitted
+     * under their key name; other types (e.g. images, binary plists) are skipped.
+     */
+    private void processQuickTimeItemList(@Nullable byte[] payload) {
+        if (payload == null) {
+            return;
+        }
+        int pos = 0;
+        while (pos + 8 <= payload.length) {
+            long entrySize = readUInt32(payload, pos);
+            if (entrySize < 8 || pos + entrySize > payload.length) {
+                return;
+            }
+            int index = (int) readUInt32(payload, pos + 4);
+            int entryEnd = (int) (pos + entrySize);
+            int data = pos + 8;
+            //inner 'data' box: size(4) type(4) valueType(4) locale(4) value
+            if (data + 16 <= entryEnd) {
+                long dataSize = readUInt32(payload, data);
+                boolean isData = payload[data + 4] == 'd' && payload[data + 5] == 'a'
+                        && payload[data + 6] == 't' && payload[data + 7] == 'a';
+                if (isData && dataSize >= 16 && data + dataSize <= entryEnd) {
+                    int valueType = (int) readUInt32(payload, data + 8);
+                    int valueLength = (int) dataSize - 16;
+                    if (index >= 1 && index <= quickTimeMetadataKeys.size()) {
+                        String key = quickTimeMetadataKeys.get(index - 1);
+                        String value = decodeValue(payload, data + 16, valueLength, valueType);
+                        if (value != null) {
+                            tikaMetadata.add(key, value);
+                            if (key.equals(QT_LOCATION_ISO6709)) {
+                                addLocation(value);
+                            }
+                        }
+                    }
+                }
+            }
+            pos += (int) entrySize;
+        }
+    }
+
+    /**
+     * Maps an ISO 6709 location string (latitude, longitude, optional altitude) to the
+     * standard {@code geo:lat}/{@code geo:long}/{@code geo:alt} properties, in addition to
+     * the raw value, so QuickTime location matches the {@code geo:*} output of the udta path.
+     */
+    private void addLocation(String iso6709) {
+        Matcher matcher = ISO6709_PATTERN.matcher(iso6709);
+        if (matcher.find()) {
+            tikaMetadata.set(TikaCoreProperties.LATITUDE, Double.parseDouble(matcher.group(1)));
+            tikaMetadata.set(TikaCoreProperties.LONGITUDE, Double.parseDouble(matcher.group(2)));
+            if (matcher.group(3) != null) {
+                tikaMetadata.set(TikaCoreProperties.ALTITUDE, Double.parseDouble(matcher.group(3)));
+            }
+        }
+    }
+
+    /**
+     * Decodes a metadata item value of one of the QTFF "well-known" types to a string,
+     * or returns null for types that are not handled (e.g. images or binary plists).
+     * Integers may be 1 to 8 bytes wide (e.g. the live-photo.auto flag is a single byte).
+     */
+    @Nullable
+    private static String decodeValue(byte[] b, int off, int len, int valueType) {
+        switch (valueType) {
+            case QT_TEXT_TYPE:
+                return new String(b, off, len, StandardCharsets.UTF_8);
+            case QT_INT_BE_TYPE:
+            case QT_UINT_BE_TYPE:
+                if (len < 1 || len > 8) {
+                    return null;
+                }
+                byte[] intBytes = Arrays.copyOfRange(b, off, off + len);
+                return valueType == QT_INT_BE_TYPE
+                        ? new BigInteger(intBytes).toString()
+                        : new BigInteger(1, intBytes).toString();
+            case QT_FLOAT32_TYPE:
+                return len == 4 ? String.valueOf(ByteBuffer.wrap(b, off, len).getFloat()) : null;
+            case QT_FLOAT64_TYPE:
+                return len == 8 ? String.valueOf(ByteBuffer.wrap(b, off, len).getDouble()) : null;
+            default:
+                return null;
+        }
+    }
+
+    private static long readUInt32(byte[] b, int off) {
+        return ((b[off] & 0xFFL) << 24) | ((b[off + 1] & 0xFFL) << 16)
+                | ((b[off + 2] & 0xFFL) << 8) | (b[off + 3] & 0xFFL);
+    }
 }

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/boxes/TikaUserDataBox.java

@@ -44,7 +44,7 @@ public class TikaUserDataBox {
     private static final String HDLR = "hdlr";
     private static final String MDIR = "mdir";//apple metadata itunes reader
     private static final Pattern COORDINATE_PATTERN =
-            Pattern.compile("([+-]\\d+\\.\\d+)([+-]\\d+\\.\\d+)");
+            Pattern.compile("([+-]\\d+\\.\\d+)([+-]\\d+\\.\\d+)([+-]\\d+(?:\\.\\d+)?)?");
 
     @Nullable
     private String coordinateString;
@@ -270,6 +270,11 @@ public void addMetadata(Mp4Directory directory) {
                 double longitude = Double.parseDouble(matcher.group(2));
                 directory.setDouble(8193, latitude);
                 directory.setDouble(8194, longitude);
+                //Mp4Directory has no altitude tag, so set geo:alt directly
+                if (matcher.group(3) != null) {
+                    metadata.set(TikaCoreProperties.ALTITUDE,
+                            Double.parseDouble(matcher.group(3)));
+                }
             }
         }
     }

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java

@@ -281,4 +281,42 @@ private Set<String> getVals(Metadata m, String k) {
         }
         return vals;
     } */
+
+    @Test
+    public void testQuickTimeMetadataKeys() throws Exception {
+        //QuickTime item-list metadata (moov/meta/keys+ilst, the com.apple.quicktime.*
+        //keys such as the content identifier and ISO 6709 location) was previously
+        //dropped by the MP4 handler. See TIKA-2861.
+        Metadata metadata = new Metadata();
+        getText("testMP4_QuickTimeMetadata.mov", metadata);
+        assertEquals("TEST-UUID-0001-LIVEPHOTO",
+                metadata.get("com.apple.quicktime.content.identifier"));
+
+        //the raw ISO 6709 location is preserved ...
+        assertEquals("+12.3456-098.7654+010.500/",
+                metadata.get("com.apple.quicktime.location.ISO6709"));
+        //... and also mapped to the standard geo:* properties (incl. altitude)
+        assertEquals(12.3456, Double.parseDouble(metadata.get(TikaCoreProperties.LATITUDE)), 0.00001);
+        assertEquals(-98.7654, Double.parseDouble(metadata.get(TikaCoreProperties.LONGITUDE)), 0.00001);
+        assertEquals(10.5, Double.parseDouble(metadata.get(TikaCoreProperties.ALTITUDE)), 0.00001);
+
+        //numeric well-known value types (uint8, float32, int32, float64)
+        assertEquals("1", metadata.get("com.apple.quicktime.live-photo.auto"));
+        assertEquals("0.75", metadata.get("com.apple.quicktime.live-photo.vitality-score"));
+        assertEquals("-13",
+                metadata.get("com.apple.quicktime.camera.focal_length.35mm_equivalent"));
+        assertEquals("1.5",
+                metadata.get("com.apple.quicktime.full-frame-rate-playback-intent"));
+    }
+
+    @Test
+    public void testUdtaLocation() throws Exception {
+        //the udta "(c)xyz" ISO 6709 location is mapped to geo:lat/geo:long, and its
+        //optional altitude, which was previously dropped, to geo:alt. See TIKA-2861.
+        Metadata metadata = new Metadata();
+        getText("testMP4_udtaLocation.mp4", metadata);
+        assertEquals(12.3456, Double.parseDouble(metadata.get(TikaCoreProperties.LATITUDE)), 0.00001);
+        assertEquals(-98.7654, Double.parseDouble(metadata.get(TikaCoreProperties.LONGITUDE)), 0.00001);
+        assertEquals(10.5, Double.parseDouble(metadata.get(TikaCoreProperties.ALTITUDE)), 0.00001);
+    }
 }