Permalink
Browse files

Address review comments from PR #5 and update docs to make it simplie…

…r to launch repack scripts.
  • Loading branch information...
1 parent 7cab968 commit 1645f2ef30c100d3c7813f79b75771ba5486d6ef @ochameau committed Feb 7, 2013
View
@@ -1,3 +1,6 @@
amo_db_config.yml
-sdks
+jetpack_data.txt
+sdks/
+src/
+xpis/
*.xpi
File renamed without changes.
View
@@ -3,41 +3,62 @@
0/ Setup environnement and download these scripts
```
# We assume that we segregate all these repack files in '/addons' folder
+# (Note that this folder can be anywhere else, it is just for sake of clarity)
+
$ mkdir /addons
$ cd /addons
-$ git clone https://github.com/ochameau/jetpack-repacker repacker
+$ git clone https://github.com/ochameau/jetpack-repacker .
- or -
$ wget --no-check-certificate https://github.com/ochameau/jetpack-repacker/tarball/master -O - | tar xz
```
1/ Download xpi files
+
+a) From AMO private FTP (suggested way, if you have access to it)
```
-$ python2.7 fetch_amo.py
+$ cd /addons
+$ cp amo_db_config.yml.example amo_db_config.yml
+$ vi amo_db_config.yml # set database credentials
+$ python2.7 scripts/fetch_amo.py
# this will fetch records from the AMO database for every current sdk-based
-# add-on and download the xpi files to ./srcipts/addons/.
+# add-on and download the xpi files to ./xpis/.
# to capture errors, grep for 'ERR' eg
-# python2.7 fetch_amo.py | grep ERR > errors.txt
+# python2.7 scripts/fetch_amo.py 2> errors.txt
+# (On 2013/02, it downloaded ~400MB of files.)
+```
+
+b) From public FTP
+```
+$ cd /addons
+$ ./scripts/fetch-ftp.sh
+# This script will download all xpi files. You can re-run it at anytime to download only new files.
+# but note that remove files from mozilla ftp will be kept locally.
+# xpi will be in /addons/ftp/ftp.mozilla.org/pub/mozilla.org/addons/
+# (On 2012/07, it downloaded 17GB of files.)
```
2/ Unzip jetpack xpi files
```
-$ /addons/repacker/scripts/unzip.sh /addons/ftp/ /addons/src
+# depending from where you downloaded xpi, run one of these scripts:
+$ cd /addons
+$ ./scripts/unzip-ftp.sh xpis/ src/
+$ ./scripts/unzip-amo.sh xpis/ src/
# This script will unzip all jetpack xpi files to /addons/src/jetpack folder
# (On 2012/07, it unpacked 770MB of data)
```
3/ Checkout all SDK released versions
```
-$ mkdir /addons/sdks
-$ /addons/sdks
-$ /addons/repacker/scripts/clone_add_sdk_versions.sh
+$ cd /addons
+$ mkdir sdks
+$ ./scripts/clone_add_sdk_versions.sh
# It will take some time as it will checkout all tagged versions on git repo
```
4/ Compute repackability
```
$ cd /addons
-$ python /addons/repacker/unpack.py --sdks /addons/sdks/ --batch repackability /addons/src/jetpack/ > repackability 2>&1
+$ python unpack.py --sdks sdks/ --batch repackability src/jetpack/ > repackability 2>&1
# This will process each addon source code and try to repack it against same SDK version
# and will tell for each addon, which one is safely repackable
```
@@ -1,8 +1,13 @@
#!/bin/bash
+P=`pwd`
+echo $P
+mkdir -p sdks/
for i in `git ls-remote -t http://github.com/mozilla/addon-sdk "refs/tags/[1-9].*" | sed 's:.*/::'`; do
if [ ! -d $i ]; then
echo "clone " $i " ..."
- git clone http://github.com/mozilla/addon-sdk -q --branch $i --depth 1 $i;
+ git clone http://github.com/mozilla/addon-sdk -q --depth 1 $P/sdks/$i
+ cd $P/sdks/$i
+ git checkout -q $i
fi
done
View
@@ -1,6 +1,6 @@
#!/usr/bin/env python
import os, re
-from sys import stdout
+from sys import stdout, stderr
from os.path import join, abspath, isfile, isdir, exists, basename
from shutil import copyfile, copytree, rmtree
from time import strftime, strptime, localtime
@@ -17,12 +17,12 @@
con = None
config_file = './amo_db_config.yml'
-download_dir = abspath('./addons')
+download_dir = abspath('./xpis')
def getYaml(path):
""" loadin' YAML files. """
if not exists(path):
- raise "YAML file doesn't exist: %s" % path
+ raise Exception("YAML file doesn't exist: %s" % path)
return load(file(path, 'r'), Loader)
@@ -44,10 +44,9 @@ def download(id, filename, download_dir, i, total_rows):
reason = ''
if hasattr(e, 'reason'):
reason = e.reason
- stdout.write("\nERR %s %s: %s %s\n" % ( e.code, reason, id, filename ))
- stdout.flush()
- err = {'code': e.code, 'reason': reason}
- return err
+ stderr.write("ERR %s %s: %s %s\n" % ( e.code, reason, id, filename ))
+ stderr.flush()
+ return
h = u.info()
totalSize = int(h["Content-Length"])
@@ -102,7 +101,8 @@ def download(id, filename, download_dir, i, total_rows):
dbConfig['database'],
);
- queries = getYaml('./queries.yml')
+ path = os.path.join(os.path.dirname(__file__), 'queries.yml')
+ queries = getYaml(path)
cur = con.cursor()
# repack_query_limit
@@ -126,15 +126,10 @@ def download(id, filename, download_dir, i, total_rows):
print "File already exists: %s" % filename
except mdb.Error, e:
- print "Error %d: %s" % (e.args[0],e.args[1])
+ stderr.write("Error %d: %s\n" % (e.args[0],e.args[1]))
sys.exit(1)
finally:
-
- if con:
+ if con:
con.close()
-
- print len(errors)
- if len(errors) > 0:
- print errors
View
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# `ftp` folder should be a clone of ftp://ftp.mozilla.org/pub/mozilla.org/addons/
+# `src` will contain unzipped addons content
+# `src/jetpack/` will contain all jetpack addons
+# `src/xul` will contain other kind of addons
+
+FTP_DIR=$1
+SRC_DIR=$2
+# AMO script only download jetpack addons
+KIND=jetpack
+
+mkdir -p $SRC_DIR/jetpack
+
+for XPI in $(ls -d $FTP_DIR/*.xpi)
+do
+ ID=$(basename $XPI)
+ # echo $XPI $KIND $ID
+ DST_DIR=$SRC_DIR/$KIND/$ID
+ mkdir -p $DST_DIR
+ unzip $XPI $DST_DIR && echo "unziped $ID" || echo "Failed to unzip $ID"
+# unzip -oq $XPI -d $DST_DIR || echo "Failed to unzip $ID"
+done
File renamed without changes.
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-for i in `./get_sdk_released_versions.sh`; do
- if [ ! -d $i ]; then
- echo "clone " $i " ..."
- git clone http://github.com/mozilla/addon-sdk -q --branch $i --depth 1 $i;
- fi
-done
@@ -1,3 +0,0 @@
-#!/bin/sh
-
-git ls-remote -t http://github.com/mozilla/addon-sdk "refs/tags/[1-9].*" | sed 's:.*/::'
View
@@ -349,10 +349,10 @@ def processAddon(path, args):
try:
bad_files = verify_addon(zip, version, manifest)
except Exception, e:
- print path + ": " + str(e)
+ print >> sys.stderr, path + ": " + str(e)
return
if not args.force and len(bad_files) > 0:
- print path + ": checksum - Unable to repack because of wrong checksum or unknown files: " + str(bad_files)
+ print >> sys.stderr, path + ": checksum - Unable to repack because of wrong checksum or unknown files: " + str(bad_files)
return
sdk_path = os.path.join(args.sdks, version)
if not os.path.exists(sdk_path):
@@ -366,16 +366,16 @@ def processAddon(path, args):
# We do not want bump either
bump=False)
except Exception, e:
- print path + ": " + str(e)
+ print >> sys.stderr, path + ": " + str(e)
return
if not repacked_path:
- print path + ": error while repacking"
+ print >> sys.stderr, path + ": error while repacking"
return
diffs = report_diff(path, repacked_path)
if len(diffs) == 0:
print path + ": repackable [" + version + "]"
else:
- print path + ": " + ", ".join(diffs)
+ print >> sys.stderr, path + ": " + ", ".join(diffs)
else:
raise Exception("Unsupported action:", args.action)
@@ -422,9 +422,9 @@ def repack(path, zip, version, manifest, target, sdk_path, force=False, useInsta
shutil.move(tmpXpiPath, xpi_path)
else:
- print "Error while building the new xpi: "
- print std[0]
- print std[1]
+ print >> sys.stderr, "Error while building the new xpi: "
+ print >> sys.stderr, std[0]
+ print >> sys.stderr, std[1]
xpi_path = False
# Delete the temporary folder
@@ -762,10 +762,10 @@ def unpack(zip, version, manifest, target, useInstallRdfId=True, bump=True):
args = parser.parse_args()
if args.action == "repack" and not args.sdk:
- print "`repack` requires --sdk option to be given."
+ print >> sys.stderr, "`repack` requires --sdk option to be given."
sys.exit()
elif args.action == "repackability" and not args.sdks:
- print "`repackability` requires --sdks option to be given."
+ print >> sys.stderr, "`repackability` requires --sdks option to be given."
sys.exit()
if args.batch:
@@ -778,7 +778,7 @@ def unpack(zip, version, manifest, target, useInstallRdfId=True, bump=True):
if os.path.isdir(path) or os.path.splitext(path)[1] == "xpi":
processAddon(path, args)
except Exception, e:
- print "Unable to", args.action, path, ": ", e
+ print >> sys.stderr, "Unable to", args.action, path, ": ", e
else:
processAddon(args.path, args)

0 comments on commit 1645f2e

Please sign in to comment.