mf2 html: switch p-size to <data> to include both integer bytes and s…

…tring

for #169

README.md

@@ -312,7 +312,7 @@ Changelog
 * Add Python 3.7 support, and improve overall Python 3 compatibility.
 * Update a number of dependencies.
 * Switch from Python's built in `json` module to [`ujson`](https://github.com/esnme/ultrajson/) to speed up JSON parsing and encoding.
-* Add `duration` and `size` support to ActivityStreams 1 and 2, RSS, and microformats2 HTML and JSON. [microformats2 support is still emerging for both](https://indieweb.org/podcast#Brainstorming). Both integer seconds and [ISO 8601 string durations](https://en.wikipedia.org/wiki/ISO_8601#Durations) are supported for `duration`. Integer bytes is used for `size` everywhere except microformats2, which uses human-readable strings, eg `5.1 MB`. ([#169](https://github.com/snarfed/granary/issues/169))
+* Add `duration` and `size` support to ActivityStreams 1 and 2, RSS, and microformats2 HTML and JSON. [microformats2 support is still emerging for both](https://indieweb.org/podcast#Brainstorming). Both integer seconds and [ISO 8601 string durations](https://en.wikipedia.org/wiki/ISO_8601#Durations) are supported for `duration`. Integer bytes is used for `size` everywhere. microformats2 HTML also includes human-readable strings, eg `5.1 MB`. ([#169](https://github.com/snarfed/granary/issues/169))
 * Twitter:
   * `[preview]_create()`: detect attempts to upload [images over 5MB](https://developer.twitter.com/en/docs/media/upload-media/uploading-media/media-best-practices#image-specs) and return an error.
 * Facebook:

granary/microformats2.py

@@ -45,6 +45,7 @@
   $content
   </div>
 $attachments
+$sizes
 $event_times
 $location
 $categories
@@ -211,9 +212,11 @@ def object_to_json(obj, trim_nulls=True, entry_class='h-entry',
 
   # prefer duration and size from object's stream, then first video, then first
   # audio
-  stream = (
-    get_first(obj, 'stream') or
-    (get_first(obj, 'video') or get_first(obj, 'audio', {}).get('stream', {})))
+  stream = {}
+  for candidate in [obj] + attachments['video'] + attachments['audio']:
+    for stream in get_list(candidate, 'stream'):
+      if stream:
+        break
 
   duration = stream.get('duration')
   if duration is not None:
@@ -223,9 +226,10 @@ def object_to_json(obj, trim_nulls=True, entry_class='h-entry',
       logging('Ignoring duration %r; expected int, got %s', duration.__class__)
       duration = None
 
+  sizes = []
   size = stream.get('size')
   if size:
-    size = humanfriendly.format_size(size)
+    sizes = [str(size)]
 
   # construct mf2!
   ret = {
@@ -245,7 +249,7 @@ def object_to_json(obj, trim_nulls=True, entry_class='h-entry',
                            get_urls(primary, 'stream')),
       'audio': get_urls(attachments, 'audio', 'stream'),
       'duration': [duration],
-      'size': [size],
+      'size': sizes,
       'published': [obj.get('published', primary.get('published', ''))],
       'updated': [obj.get('updated', primary.get('updated', ''))],
       'in-reply-to': util.trim_nulls([o.get('url') for o in in_reply_tos]),
@@ -462,14 +466,8 @@ def is_absolute(url):
         logging.warning('Unknown format for length or duration %r', duration)
 
 
-  size = prop.get('size')
-  if size:
-    try:
-      size = humanfriendly.parse_size(size)
-    except humanfriendly.InvalidSize:
-      logging.warning("Couldn't parse size %r", size)
-
   stream = None
+  bytes = size_to_bytes(prop.get('size'))
   for type in 'audio', 'video':
     atts = [{
       'objectType': type,
@@ -478,7 +476,7 @@ def is_absolute(url):
         # integer seconds: http://activitystrea.ms/specs/json/1.0/#media-link
         'duration': duration,
         # file size in bytes. nonstandard, not in AS1 or AS2
-        'size': size,
+        'size': bytes,
       },
     } for url in get_string_urls(props.get(type, []))]
     attachments.extend(atts)
@@ -719,13 +717,23 @@ def json_to_html(obj, parent_props=None):
   summary = ('<div class="p-summary">%s</div>' % prop.get('summary')
              if prop.get('summary') else '')
 
-  # render attachments
+  # attachments
   # TODO: use photo alt property as alt text once mf2py handles that.
   # https://github.com/tommorris/mf2py/issues/83
   attachments = []
   for name, fn in ('photo', img), ('video', vid), ('audio', aud):
     attachments.extend(fn(val) for val in props.get(name, []))
 
+  # size(s)
+  # https://github.com/snarfed/granary/issues/169#issuecomment-547918405
+  sizes = []
+  for size in props.get('size', []):
+    bytes = size_to_bytes(size)
+    if size:
+      sizes.append('<data class="p-size" value="%s">%s</data>' %
+                   (bytes, humanfriendly.format_size(bytes)))
+
+  # categories
   cats = props.get('category', [])
   people = [
     hcard_to_html(cat, ['u-category', 'h-card']) for cat in cats
@@ -776,6 +784,7 @@ def json_to_html(obj, parent_props=None):
     location=hcard_to_html(location, ['p-location']),
     categories='\n'.join(people + tags),
     attachments='\n'.join(attachments),
+    sizes='\n'.join(sizes),
     links='\n'.join(links),
     invitees='\n'.join([hcard_to_html(i, ['p-invitee'])
                         for i in props.get('invitee', [])]),
@@ -1231,3 +1240,24 @@ def maybe_datetime(str, classname):
     return '<time class="%s" datetime="%s">%s</time>' % (classname, str, str)
   else:
     return ''
+
+
+def size_to_bytes(size):
+  """Converts a string file size to an integer number of bytes.
+
+  Args:
+    size, string, may be either integer bytes or human-readable approximation,
+      eg 7MB or 1.23 kb
+
+  Returns: integer, bytes, or None if size can't be parsed
+  """
+  if util.is_int(size):
+    return int(size)
+
+  if not size:
+    return None
+
+  try:
+    return humanfriendly.parse_size(size)
+  except humanfriendly.InvalidSize:
+    logging.debug("Couldn't parse size %r", size)

granary/tests/test_microformats2.py

@@ -890,3 +890,15 @@ def test_html_to_activities_brs_to_newlines(self):
       'content_is_html': True,
       'displayName': 'foo bar\nbaz \n\n baj',
     }}], activities)
+
+  def test_size_to_bytes(self):
+    for input, expected in (
+        (None, None),
+        ('', None),
+        (123, 123),
+        ('123', 123),
+        ('  123\n  ', 123),
+        ('1.23 KB', 1230),
+        ('5 MB', 5000000),
+    ):
+      self.assertEqual(expected, microformats2.size_to_bytes(input), input)

granary/tests/testdata/article_with_audio_video.as.json

@@ -7,14 +7,14 @@
     "stream": {
       "url": "http://a/podcast.mp3",
       "duration": 123,
-      "size": 45600
+      "size": 45678
     }
   }, {
     "objectType": "video",
     "stream": {
       "url": "http://a/vidjo.mov",
       "duration": 123,
-      "size": 45600
+      "size": 45678
     }
   }]
 }

granary/tests/testdata/article_with_audio_video.as2.json

@@ -7,11 +7,11 @@
     "type": "Audio",
     "url": "http://a/podcast.mp3",
     "duration": "P0DT123S",
-    "size": 45600
+    "size": 45678
   }, {
     "type": "Video",
     "url": "http://a/vidjo.mov",
     "duration": "P0DT123S",
-    "size": 45600
+    "size": 45678
   }]
 }

granary/tests/testdata/article_with_audio_video.mf2-from-as.json

@@ -6,6 +6,6 @@
     "audio": ["http://a/podcast.mp3"],
     "video": ["http://a/vidjo.mov"],
     "duration": ["123"],
-    "size": ["45600"]
+    "size": ["45678"]
   }
 }

granary/tests/testdata/article_with_audio_video.mf2.html

@@ -8,5 +8,6 @@
 
   <video class="u-video" src="http://a/vidjo.mov" controls="controls" poster="">Your browser does not support the video tag. <a href="http://a/vidjo.mov">Click here to view directly. </a></video>
   <audio class="u-audio" src="http://a/podcast.mp3" controls="controls">Your browser does not support the audio tag. <a href="http://a/podcast.mp3">Click here to listen directly.</a></audio>
+  <data class="p-size" value="45678">45.68 KB</data>
 
 </article>

granary/tests/testdata/article_with_audio_video.mf2.json

@@ -5,10 +5,11 @@
     "video": ["http://a/vidjo.mov"],
     "audio": ["http://a/podcast.mp3"],
     "length": ["123"],
-    "size": ["45.6 KB"],
+    "size": ["45678"],
     "content": [{
       "html": "something",
       "value": "something"
     }]
   }
 }
+

granary/tests/testdata/note_with_audio_video.mf2-from-as.json

@@ -5,6 +5,6 @@
     "audio": ["http://a/podcast.mp3"],
     "video": ["http://a/vidjo.mov"],
     "duration": ["123"],
-    "size": ["4.57 KB"]
+    "size": ["4567"]
   }
 }