Get west and manifest revisions from a configuration file

Add configuration file support via the standard Python configparser
module, which uses a Git-like .ini format. Use it to save the west and
manifest revisions specified when running 'west init', and use the saved
revisions when updating the west and manifest repositories.

Configuration file paths are based on Git, for consistency with the
Git-like commands. See the FILES section in git-config(1). Mac OS and
Windows paths were checked by Carles Cufi with
'git config --list --show-origin'.

Add some new shorthands to _expand_shorthands() in project.py to factor
out some logic while removing the hardcoding of the west and manifest
upstream branches. The qual-* versions of shorthands give the full path
to refs (refs/heads/foo, remotes/origin/bar, etc.)

Fixes: #63
Fixes: #67

Signed-off-by: Ulf Magnusson <Ulf.Magnusson@nordicsemi.no>

src/west/_bootstrap/main.py

@@ -6,6 +6,7 @@
 '''
 
 import argparse
+import configparser
 import os
 import platform
 import subprocess
@@ -195,6 +196,14 @@ def init_bootstrap(directory, args):
     clone(args.manifest_url, args.manifest_rev,
           os.path.join(directory, WEST_DIR, MANIFEST))
 
+    # Create an initial configuration file
+
+    config = configparser.ConfigParser()
+    config['west'] = {'revision': args.west_rev}
+    config['manifest'] = {'revision': args.manifest_rev}
+    with open(os.path.join(directory, WEST_DIR, 'config'), 'w') as f:
+        config.write(f)
+
     # Create a dotfile to mark the installation. Hide it on Windows.
 
     with open(os.path.join(directory, WEST_DIR, WEST_MARKER), 'w') as f:

src/west/commands/project.py

@@ -11,6 +11,7 @@
 import subprocess
 import textwrap
 
+from west import config
 from west import log
 from west import util
 from west.commands import WestCommand
@@ -492,13 +493,7 @@ def _fetch(project):
         #
         # Note: Many servers won't allow fetching arbitrary commits by SHA.
         # Combining --depth with an SHA will break for those.
-
-        # Qualify branch names with refs/heads/, just to be safe. Just the
-        # branch name is likely to work as well though.
-        _git(project,
-             'fetch --depth=(clone-depth) origin ' +
-                 (project.revision if _is_sha(project.revision) else \
-                     'refs/heads/' + project.revision))
+        _git(project, 'fetch --depth=(clone-depth) origin (qual-rev)')
 
     else:
         _inf(project, 'Fetching changes for (name-and-path)')
@@ -509,10 +504,11 @@ def _fetch(project):
 
     # Create/update the 'manifest-rev' branch
     _git(project,
-         'update-ref refs/heads/(manifest-rev-branch) ' +
-             (project.revision if _is_sha(project.revision) else
-                 'remotes/origin/' + project.revision))
+         'update-ref (qual-manifest-rev-branch) (qual-remote-rev)')
 
+    # TODO: Check if the repository is in the post-'git init' state instead.
+    # That allows repositories to be properly initialized even if the initial
+    # 'west fetch' is aborted and then resumed.
     if not exists:
         # If we just initialized the repository, check out 'manifest-rev' in a
         # detached HEAD state.
@@ -525,14 +521,14 @@ def _fetch(project):
         # (The --detach flag is strictly redundant here, because the
         # refs/heads/<branch> form already detaches HEAD, but it avoids a
         # spammy detached HEAD warning from Git.)
-        _git(project, 'checkout --detach refs/heads/(manifest-rev-branch)')
+        _git(project, 'checkout --detach (qual-manifest-rev-branch)')
 
     return exists
 
 
 def _rebase(project):
     _inf(project, 'Rebasing (name-and-path) to (manifest-rev-branch)')
-    _git(project, 'rebase (manifest-rev-branch)')
+    _git(project, 'rebase (qual-manifest-rev-branch)')
 
 
 def _cloned(project):
@@ -578,7 +574,8 @@ def _create_branch(project, branch):
     else:
         _inf(project, "Creating branch '{}' in (name-and-path)"
                       .format(branch))
-        _git(project, 'branch --quiet --track {} (manifest-rev-branch)'
+
+        _git(project, 'branch --quiet --track {} (qual-manifest-rev-branch)'
                       .format(branch))
 
 
@@ -597,9 +594,11 @@ def _special_project(name):
     # so that we can reuse the project-related functions for them
     remote = Remote(name='dummy name for {} repository'.format(name),
                     url='dummy URL for {} repository'.format(name))
+
+    # 'revision' always exists and defaults to 'master'
     return Project(name, remote, None,
-                   revision='master',  # FIXME this must be generalized!
-                   path=os.path.join('west', name.lower()))
+                   revision=config.config[name]['revision'],
+                   path=os.path.join('west', name))
 
 
 def _update(update_west, update_manifest):
@@ -614,7 +613,7 @@ def attempt(project, cmd):
 
     projects = []
     if update_west:
-        projects.append(_special_project('West'))
+        projects.append(_special_project('west'))
     if update_manifest:
         projects.append(_special_project('manifest'))
 
@@ -625,14 +624,14 @@ def attempt(project, cmd):
         attempt(project, 'fetch')
 
         # Get the SHA of the last commit in common with the upstream branch
-        merge_base = attempt(project, 'merge-base HEAD remotes/origin/master')
+        merge_base = attempt(project, 'merge-base HEAD (qual-remote-rev)')
 
         # Get the current SHA of the upstream branch
-        head_sha = attempt(project, 'show-ref --hash remotes/origin/master')
+        head_sha = attempt(project, 'show-ref --hash (qual-remote-rev)')
 
         # If they differ, we need to rebase
         if merge_base != head_sha:
-            attempt(project, 'rebase remotes/origin/master')
+            attempt(project, 'rebase (qual-remote-rev)')
 
             _inf(project, 'Updated (rebased) (name-and-path) to the '
                           'latest version')
@@ -754,15 +753,40 @@ def _expand_shorthands(project, s):
     # Expands project-related shorthands in 's' to their values,
     # returning the expanded string
 
+    # Some of the trickier ones below. 'qual' stands for 'qualified', meaning
+    # the full path to the ref (e.g. refs/heads/master).
+    #
+    # qual-rev:
+    #   The qualified local branch for the revision from the manifest (e.g.
+    #   refs/heads/master), or an SHA if the revision is an SHA
+    #
+    # qual-remote-rev:
+    #   The qualified remote branch for the revision (e.g.
+    #   remotes/origin/master), or an SHA if the revision is an SHA
+    #
+    # manifest-rev-branch:
+    #   The name of the magic branch that points to the manifest revision
+    #
+    # qual-manifest-rev-branch:
+    #   A qualified reference to the magic manifest revision branch, e.g.
+    #   refs/heads/manifest-rev
+
     return s.replace('(name)', project.name) \
             .replace('(name-and-path)',
                      '{} ({})'.format(
                          project.name, os.path.join(project.path, ""))) \
             .replace('(url)', project.url) \
             .replace('(path)', project.path) \
             .replace('(abspath)', project.abspath) \
-            .replace('(revision)', project.revision) \
+            .replace('(qual-rev)',
+                     project.revision if _is_sha(project.revision) else \
+                     'refs/heads/' + project.revision) \
+            .replace('(qual-remote-rev)',
+                     project.revision if _is_sha(project.revision) else \
+                     'remotes/origin/' + project.revision) \
             .replace('(manifest-rev-branch)', _MANIFEST_REV_BRANCH) \
+            .replace('(qual-manifest-rev-branch)',
+                     'refs/heads/' + _MANIFEST_REV_BRANCH) \
             .replace('(clone-depth)', str(project.clone_depth))
 
 

src/west/config.py

@@ -0,0 +1,54 @@
+'''
+Configuration file handling, using the standard configparser module.
+'''
+
+import configparser
+import os
+import platform
+
+from west.util import west_dir
+
+def read_config():
+    '''
+    Reads all configuration files, making the configuration values available as
+    a configparser.ConfigParser object in config.config. This object works
+    similarly to a dictionary: config.config['foo']['bar'] gets the value for
+    key 'bar' in section 'foo'.
+    '''
+
+    global config
+
+    #
+    # Gather (potential) configuration file paths, roughly following Git
+    # conventions.
+    #
+    # See the FILES section in git-config(1).
+    #
+
+    # System-wide
+
+    if platform.system() == 'Linux':
+        files = ['/etc/westconfig']
+    elif platform.system() == 'Darwin':  # Mac OS
+        # This was seen on Carles' machine ($(prefix) = /usr/local)
+        files = ['/usr/local/etc/westconfig']
+    elif platform.system() == 'Windows':
+        # Seen on Carles' machine. Not sure how it got derived though.
+        files = [os.path.expandvars('%PROGRAMDATA%\\west\\config')]
+
+    # Probably doesn't hurt to check $XDG_CONFIG_HOME on all systems
+    files.append(os.path.expandvars('$XDG_CONFIG_HOME/git/config')
+                 if 'XDG_CONFIG_HOME' in os.environ else
+                 os.path.expanduser('~/.config/git/config'))
+
+    # User-specific and repository-specific
+
+    files.append(os.path.expanduser('~/.westconfig'))
+    files.append(os.path.join(west_dir(), 'config'))
+
+    #
+    # Parse all existing configuration files. Also give any defaults.
+    #
+
+    config = configparser.ConfigParser(defaults={"revision": "master"})
+    config.read(files, encoding='utf-8')

src/west/main.py

@@ -12,11 +12,11 @@
 import colorama
 from functools import partial
 import os
-import platform
 import sys
 from subprocess import CalledProcessError, check_output, DEVNULL
 
 from west import log
+from west import config
 from west.commands import CommandContextError
 from west.commands.build import Build
 from west.commands.flash import Flash
@@ -25,7 +25,7 @@
                              Checkout, Diff, Status, Update, ForAll, \
                              WestUpdated
 from west.manifest import Manifest
-from west.util import quote_sh_list, in_multirepo_install
+from west.util import quote_sh_list, in_multirepo_install, west_dir
 
 IN_MULTIREPO_INSTALL = in_multirepo_install(__file__)
 
@@ -199,6 +199,9 @@ def main(argv=None):
         argv = sys.argv[1:]
     args, unknown = parse_args(argv)
 
+    # Read the configuration files
+    config.read_config()
+
     for_stack_trace = 'run as "west -v ... {} ..." for a stack trace'.format(
         args.command)
     try: