Stackbrew overhaul: v2 (WIP)

stackbrew/app.py

@@ -5,7 +5,7 @@
 
 sys.path.append('./lib')
 
-import brew
+import brew.v2 as brew
 import db
 import periodic
 import utils
@@ -15,6 +15,8 @@
 with open('./config.json') as config_file:
     config = json.load(config_file)
 data = db.DbManager(config['db_url'], debug=config['debug'])
+brew.logger = app.logger
+brew.set_loglevel('DEBUG' if config['debug'] else 'INFO')
 
 
 @app.route('/')
@@ -49,12 +51,16 @@ def force_build():
 
 
 def build_task():
-    summary = brew.build_library(
-        config['library_repo'], namespace='stackbrew',
-        debug=config['debug'], push=config['push'], prefill=False,
-        repos_folder=config['repos_folder'], logger=app.logger
+    summary = data.new_summary()
+    library = brew.StackbrewLibrary(config['library_repo'])
+    builder = brew.LocalBuilder(
+        library=library, namespaces=config['namespaces'],
+        repo_cache=config['repos_folder']
     )
-    data.insert_summary(summary)
+    builder.build_repo_list()
+    builder.build_all(callback=summary.handle_build_result)
+    if config['push']:
+        builder.push_all()
 
 
 try:

stackbrew/brew-cli

@@ -27,10 +27,10 @@ if __name__ == '__main__':
         '--debug', default=False, action='store_true',
         help='Enable debugging output'
     )
-    parser.add_argument(
-        '--noprefill', default=True, action='store_false',
-        dest='prefill', help='Disable cache prefill'
-    )
+#    parser.add_argument(
+#        '--noprefill', default=True, action='store_false',
+#        dest='prefill', help='Disable cache prefill'
+#    )
     parser.add_argument(
         '-n', metavar='NAMESPACE', default='library',
         help='Namespace used for generated repositories. Default is library'
@@ -45,17 +45,25 @@ if __name__ == '__main__':
         nargs='?', help='git repository containing the library definition'
         ' files. Default is ' + brew.DEFAULT_REPOSITORY
     )
-    parser.add_argument(
-        '--reg', default=None, help='Registry address to'
-        ' push build results to. Also sets push to true.'
-    )
+#    parser.add_argument(
+#        '--reg', default=None, help='Registry address to'
+#        ' push build results to. Also sets push to true.'
+#    )
     parser.add_argument(
         '--targets', default=None, help='Comma-separated list'
         ' of images to build.'
     )
     args = parser.parse_args()
-    summary = brew.build_library(
-        args.repository, args.b, args.n, args.push or args.reg is not None,
-        args.debug, args.prefill, args.reg, args.targets, None, logger
-    )
-    sys.exit(summary.exit_code())
+    if args.debug:
+        brew.v2.set_loglevel('DEBUG')
+    sb_library = brew.v2.StackbrewLibrary(args.repository, args.b)
+    builder = brew.v2.LocalBuilder(sb_library, args.n.split(','), args.targets)
+    builder.build_repo_list()
+    builder.build_all()
+    if args.push:
+        builder.push_all()
+
+#    summary = brew.build_library(
+#        args.repository, args.b, args.n, args.push or args.reg is not None,
+#        args.debug, args.prefill, args.reg, args.targets, None, logger
+#    )

stackbrew/brew/__init__.py

@@ -1 +1,2 @@
 from brew import build_library, DEFAULT_REPOSITORY, DEFAULT_BRANCH
+from . import v2

stackbrew/brew/brew.py

@@ -56,6 +56,8 @@ def build_library(repository=None, branch=None, namespace=None, push=False,
         branch = DEFAULT_BRANCH
     if debug:
         logger.setLevel('DEBUG')
+    else:
+        logger.setLevel('INFO')
     if targetlist is not None:
         targetlist = targetlist.split(',')
 

stackbrew/brew/git.py

@@ -1,15 +1,34 @@
+import logging
 import os
+import subprocess
 import tempfile
-import logging
 
-from dulwich import index
-from dulwich.client import get_transport_and_path
-from dulwich.objects import Tag
-from dulwich.repo import Repo
 
 logger = logging.getLogger(__name__)
 
 
+class GitException(Exception):
+    pass
+
+
+class Repo(object):
+    def __init__(self, path, repo_url):
+        self.path = path
+        self.repo_url = repo_url
+
+    def clone(self):
+        logger.debug('Cloning {0} into {1}'.format(self.repo_url, self.path))
+        result = _execute('clone', [self.repo_url, '.'], self.path)
+        if result != 0:
+            raise GitException('git clone failed')
+
+
+def _execute(command, args, cwd):
+    cmd = ['git', command] + args
+    logger.debug('Executing "{0}" in {1}'.format(' '.join(cmd), cwd))
+    return subprocess.Popen(cmd, cwd=cwd).wait()
+
+
 def clone_branch(repo_url, branch="master", folder=None):
     return clone(repo_url, 'refs/heads/' + branch, folder)
 
@@ -21,53 +40,36 @@ def clone_tag(repo_url, tag, folder=None):
 def checkout(rep, ref=None):
     if ref is None:
         ref = 'refs/heads/master'
-    elif ref.startswith('refs/tags'):
-        ref = rep.ref(ref)
-    if isinstance(rep[ref], Tag):
-        rep['HEAD'] = rep[ref].object[1]
-    else:
-        rep['HEAD'] = rep.refs[ref]
-    indexfile = rep.index_path()
-    tree = rep["HEAD"].tree
-    index.build_index_from_tree(rep.path, indexfile, rep.object_store, tree)
+    logger.debug("Checkout ref:{0} in {1}".format(ref, rep.path))
+    result = _execute('checkout', [ref], rep.path)
+
+    if result != 0:
+        raise GitException('git checkout failed')
+
     return rep.path
 
 
 def pull(origin, rep, ref=None):
-    clone(origin, ref, None, rep)
+    if ref is None:
+        ref = 'refs/heads/master'
+    logger.debug("Pull ref:{0} in {1}".format(ref, rep.path))
+    result = _execute('pull', ['origin', ref], rep.path)
+    if result != 0:
+        raise GitException('git pull failed')
+    checkout(rep, ref)
     return rep, rep.path
 
 
 def clone(repo_url, ref=None, folder=None, rep=None):
     if ref is None:
         ref = 'refs/heads/master'
-    logger.debug("clone repo_url={0}, ref={1}".format(repo_url, ref))
-    if not rep:
-        if folder is None:
-            folder = tempfile.mkdtemp()
-        else:
-            os.mkdir(folder)
-        logger.debug("folder = {0}".format(folder))
-        rep = Repo.init(folder)
-    client, relative_path = get_transport_and_path(repo_url)
-    logger.debug("client={0}".format(client))
-
-    remote_refs = client.fetch(relative_path, rep)
-    for k, v in remote_refs.iteritems():
-        try:
-            rep.refs.add_if_new(k, v)
-        except:
-            pass
-
-    if ref.startswith('refs/tags'):
-        ref = rep.ref(ref)
-
-    if isinstance(rep[ref], Tag):
-        rep['HEAD'] = rep[ref].object[1]
+    logger.debug("Cloning repo_url={0}, ref={1}".format(repo_url, ref))
+    if folder is None:
+        folder = tempfile.mkdtemp()
     else:
-        rep['HEAD'] = rep[ref]
-    indexfile = rep.index_path()
-    tree = rep["HEAD"].tree
-    index.build_index_from_tree(rep.path, indexfile, rep.object_store, tree)
-    logger.debug("done")
+        os.mkdir(folder)
+    logger.debug("folder = {0}".format(folder))
+    rep = Repo(folder, repo_url)
+    rep.clone()
+
     return rep, folder