4chan-to-text

#!/usr/bin/env python3

#==============================================================================
# Functionality
#==============================================================================
import pdb
import sys
import os
import re

# utility funcs, classes, etc go here.

def asserting(cond):
    if not cond:
        pdb.set_trace()
    assert(cond)

def has_stdin():
    return not sys.stdin.isatty()

def reg(pat, flags=0):
    return re.compile(pat, re.VERBOSE | flags)

#==============================================================================
# Cmdline
#==============================================================================
import argparse

parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, 
    description="""
TODO
""")
     
parser.add_argument('-v', '--verbose',
    action="store_true",
    help="verbose output" )

args = None

#==============================================================================
# Main
#==============================================================================

import sys
import json
import html2text
import ftfy
import requests
from pprint import pprint as pp

h = html2text.HTML2Text()
h.body_width = 0
h.emphasis_mark = '*'


def process(line):
  post = json.loads(line)
  if post is None:
    return
  if 'posts' in post:
    for item in post['posts']:
      process(json.dumps(item))
    return
  text = ''
  com = post.get("com", '')
  if com is not None:
    com = h.handle(com)
    com = ftfy.fix_text(com)
  post['com'] = com
  if 'name' not in post:
    post['name'] = 'Anonymous'
  if 'board' not in post:
    post['board'] = ''
  else:
    post['board'] = '/' + post['board'] + '/ '
  if post.get('filename', None):
    post['filename'] = post['filename'] + post['ext'] + '\n'
  else:
    post['filename'] = ''

  #pp(post)
  text = """
{board}{name} {now} No.{no} ▶
{filename}{com}
""".format(**post).strip() + '\n'
  print(text)


def run():
    if args.verbose:
        print(args)
    for url in args.args:
      url = url.split('#', 1)[0] # get rid of any anchor
      url = url.split('.json', 1)[0] # get rid of any .json extension
      url = url + '.json' # try adding .json and fetch.
      i = requests.get(url)
      if not i.ok:
        i.raise_for_status()
      else:
        process(i.text)
    if has_stdin():
        for line in sys.stdin:
          process(line)

def main():
    try:
        global args
        if not args:
            args, leftovers = parser.parse_known_args()
            args.args = leftovers
        return run()
    except IOError:
        # http://stackoverflow.com/questions/15793886/how-to-avoid-a-broken-pipe-error-when-printing-a-large-amount-of-formatted-data
        try:
            sys.stdout.close()
        except IOError:
            pass
        try:
            sys.stderr.close()
        except IOError:
            pass

if __name__ == "__main__":
    main()