update-firefox-version

#!/usr/bin/env python3
#
# update-firefox-version: A tool to generate extra-data entries in the
# manifest for the new Firefox version
#
# Copyright (C) 2017 Endless Mobile, Inc.
#
# Authors:
#       Joaquim Rocha <jrocha@endlessm.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

try:
    from bs4 import BeautifulSoup
except ImportError:
    print('Please install BeautifulSoup! (e.g., `sudo pip3 install bs4`)')
    exit(1)

from collections import OrderedDict
import hashlib
import json
import logging
import os
import urllib.error
import urllib.parse
import urllib.request

CURRENT_VERSION = '57.0'

class FirefoxJsonGenerator():
    JSON_FILE_NAME = 'org.mozilla.Firefox.json.in'
    MOZILLA_ARCHIVE_URL = 'https://archive.mozilla.org/pub/firefox/releases/{version}/linux-x86_64/'

    def __init__(self, version):
        logging.basicConfig(level=logging.INFO)
        self._version = version
        self._json_data = None
        self._versioned_url = self.MOZILLA_ARCHIVE_URL.format(version=self._version)

        with open(self.JSON_FILE_NAME) as json_file:
            self._json_data = json.load(json_file, object_pairs_hook=OrderedDict)
            self._json_data['finish-args'] = [arg for arg in self._json_data.get('finish-args', []) if not arg.startswith('--extra-data=')]

    def _get_extra_data_info(self, url):
        '''Fetches the extra data info for the given URL

        The extra data info fetched is the sha256 checksum, and size.
        This is needed together with the URL and name of the asset for
        the Flatpak json manifest.
        '''
        info = {}
        request = urllib.request.Request(url)

        print('Fetching info for "{}"'.format(url))

        try:
            with urllib.request.urlopen(request) as response:
                print('Loading')
                contents = b''
                size=int(response.getheader('Content-Length'))
                while not response.closed:
                    data = response.read(250000)
                    if len(data) == 0:
                        break

                    contents += data
                    progress_str = '{}/{} KB'.format(int(len(contents) / 1000),
                                                     int(size / 1000))
                    print('{:>20}\r'.format(progress_str), end='', flush=True)

                print('\n........ done')
                info['sha256sum'] = hashlib.sha256(contents).hexdigest()
                info['url'] = request.get_full_url()
                info['name'] = os.path.basename(request.get_full_url())
                info['size'] = len(contents)

                return info
        except urllib.error.HTTPError as error:
            logging.error('{}: {}'.format(url, error.msg))
            exit(1)

    def _fetch_firefox_info(self):
        '''Fetches the info for this Firefox version'''
        url = '{url}en-US/firefox-{version}.tar.bz2'.format(url=self._versioned_url,
                                                            version=self._version)
        info = self._get_extra_data_info(url)
        info['name'] = 'firefox.tar.bz2'
        return info

    def _fetch_languages_info(self):
        '''Fetches the info for all the languages add-ons in this Firefox version.'''
        url = '{url}xpi/'.format(url=self._versioned_url)
        request = urllib.request.Request(url)
        languages_html = None

        try:
            with urllib.request.urlopen(request) as response:
                languages_html = str(response.read())
        except urllib.error.HTTPError as error:
            logging.error('{}: {}'.format(url, error.msg))
            exit(1)

        languages = []
        soup = BeautifulSoup(languages_html, 'html.parser')

        for link in soup.find_all('a'):
            if not link.string.endswith('.xpi') or not 'href' in link.attrs:
                continue
            lang_link = link.get('href')
            # Make sure we add the full URL for the language add-ons
            languages.append(urllib.parse.urljoin(url, lang_link))

        info = []
        for lang in languages:
            info.append(self._get_extra_data_info(lang))
        return info

    def generate(self):
        '''
        Generate the json file from json.in adding the extra-data for Firefox
        and its languages.
        '''
        extra_data_info = [self._fetch_firefox_info()] + self._fetch_languages_info()
        finish_args = self._json_data.get('finish-args', [])
        for info in extra_data_info:
            finish_args.append('--extra-data={name}:{sha256sum}:{size}::{url}'.format(**info))
        with open(self.JSON_FILE_NAME, 'w') as f:
            f.write(json.dumps(self._json_data, indent=2))
            f.write("\n")

if __name__ == '__main__':
    generator = FirefoxJsonGenerator(CURRENT_VERSION)
    generator.generate()