Merge branch 'master' into nine

Conflicts:
	master/buildbot/test/unit/test_master.py

master/buildbot/status/buildset.py

@@ -77,6 +77,8 @@ def asDict(self):
 
 class BuildSetSummaryNotifierMixin:
 
+    _buildsetCompleteConsumer = None
+
     def summarySubscribe(self):
         self._buildsetCompleteConsumer = self.master.mq.startConsuming(
             self._buildsetComplete,

master/buildbot/steps/mswin.py

@@ -29,7 +29,7 @@ class Robocopy(ShellCommand):
     This is just a wrapper around the standard shell command that
     will handle arguments and return codes accordingly for Robocopy.
     """
-    renderables = ['source', 'destination', 'files', 'exclude']
+    renderables = ['source', 'destination', 'files', 'exclude_files', 'exclude_dirs', 'custom_opts']
 
     # Robocopy exit flags (they are combined to make up the exit code)
     # See: http://ss64.com/nt/robocopy-exit.html
@@ -45,6 +45,9 @@ def __init__(self, source, destination,
                  mirror=False,
                  move=False,
                  exclude=None,
+                 exclude_files=None,
+                 exclude_dirs=None,
+                 custom_opts=None,
                  verbose=False,
                  **kwargs):
         self.source = source
@@ -53,7 +56,11 @@ def __init__(self, source, destination,
         self.recursive = recursive
         self.mirror = mirror
         self.move = move
-        self.exclude = exclude
+        self.exclude_files = exclude_files
+        if exclude and not exclude_files:
+            self.exclude_files = exclude
+        self.exclude_dirs = exclude_dirs
+        self.custom_opts = custom_opts
         self.verbose = verbose
         ShellCommand.__init__(self, **kwargs)
 
@@ -67,11 +74,16 @@ def start(self):
             command.append('/MIR')
         if self.move:
             command.append('/MOVE')
-        if self.exclude:
+        if self.exclude_files:
             command.append('/XF')
-            command += self.exclude
+            command += self.exclude_files
+        if self.exclude_dirs:
+            command.append('/XD')
+            command += self.exclude_dirs
         if self.verbose:
             command.append('/V /TS /FP')
+        if self.custom_opts:
+            command += self.custom_opts
         command += ['/TEE', '/NP']
         self.setCommand(command)
         ShellCommand.start(self)

master/buildbot/steps/source/repo.py

@@ -94,7 +94,7 @@ class Repo(Source):
     name = 'repo'
     renderables = ["manifestURL", "manifestBranch", "manifestFile", "tarball", "jobs",
                    "syncAllBranches", "updateTarballAge", "manifestOverrideUrl",
-                   "repoDownloads"]
+                   "repoDownloads", "depth"]
 
     ref_not_found_re = re.compile(r"fatal: Couldn't find remote ref")
     cherry_pick_error_re = re.compile(r"|".join([r"Automatic cherry-pick failed",
@@ -116,6 +116,7 @@ def __init__(self,
                  updateTarballAge=7 * 24.0 * 3600.0,
                  manifestOverrideUrl=None,
                  repoDownloads=None,
+                 depth=0,
                  **kwargs):
         """
         @type  manifestURL: string
@@ -143,6 +144,9 @@ def __init__(self,
         @type repoDownloads: list of strings
         @param repoDownloads: optional repo download to perform after the repo sync
 
+        @type depth: integer
+        @param depth: optional depth parameter to repo init.
+                          If specified, create a shallow clone with given depth.
         """
         self.manifestURL = manifestURL
         self.manifestBranch = manifestBranch
@@ -155,6 +159,7 @@ def __init__(self,
         if repoDownloads is None:
             repoDownloads = []
         self.repoDownloads = repoDownloads
+        self.depth = depth
         Source.__init__(self, **kwargs)
 
         assert self.manifestURL is not None
@@ -273,7 +278,8 @@ def doRepoSync(self, forceClobber=False):
         yield self._repoCmd(['init',
                              '-u', self.manifestURL,
                              '-b', self.manifestBranch,
-                             '-m', self.manifestFile])
+                             '-m', self.manifestFile,
+                             '--depth', str(self.depth)])
 
         if self.manifestOverrideUrl:
             self.stdio_log.addHeader("overriding manifest with %s\n" % (self.manifestOverrideUrl))

master/buildbot/test/fake/fakemaster.py

@@ -49,6 +49,9 @@ def mkref(x):
         d.addCallback(mkref)
         return d
 
+    def put(self, key, val):
+        pass
+
 
 class FakeCaches(object):
 

master/buildbot/test/unit/test_master.py

@@ -166,7 +166,7 @@ def test_startup_bad_config(self):
 
         @d.addCallback
         def check(_):
-            reactor.stop.assert_called()
+            reactor.stop.assert_called_with()
             self.assertLogged("oh noes")
         return d
 
@@ -182,7 +182,7 @@ def db_setup():
 
         @d.addCallback
         def check(_):
-            reactor.stop.assert_called()
+            reactor.stop.assert_called_with()
             self.assertLogged("GOT HERE")
         return d
 
@@ -198,7 +198,7 @@ def db_setup():
 
         @d.addCallback
         def check(_):
-            reactor.stop.assert_called()
+            reactor.stop.assert_called_with()
             self.assertEqual(len(self.flushLoggedErrors(RuntimeError)), 1)
         return d
 
@@ -232,7 +232,7 @@ def test_reconfig(self):
 
         @d.addCallback
         def check(_):
-            self.master.reconfigService.assert_called()
+            self.master.reconfigService.assert_called_with(mock.ANY)
         return d
 
     @defer.inlineCallbacks

master/buildbot/test/unit/test_scripts_create_master.py

@@ -239,5 +239,5 @@ def test_createDB(self):
             mkconfig(basedir='test', quiet=True),
             _noMonkey=True)
         setup.asset_called_with(check_version=False, verbose=False)
-        upgrade.assert_called()
+        upgrade.assert_called_with()
         self.assertWasQuiet()

master/buildbot/test/unit/test_scripts_upgrade_master.py

@@ -262,5 +262,5 @@ def test_upgradeDatabase(self):
             mkconfig(basedir='test', quiet=True),
             config_module.MasterConfig())
         setup.asset_called_with(check_version=False, verbose=False)
-        upgrade.assert_called()
+        upgrade.assert_called_with()
         self.assertWasQuiet()

master/buildbot/test/unit/test_status_buildset.py

@@ -97,3 +97,7 @@ def check(_):
             self.assertEqual(builds, info['builds'])
 
             self.assertEqual(buildset['bsid'], info['bsid'])
+
+    def test_unsub_with_no_sub(self):
+        notifier = BuildSetSummaryNotifierMixin()
+        notifier.summaryUnsubscribe()

master/buildbot/test/unit/test_steps_mswin.py

@@ -99,6 +99,20 @@ def test_exclude(self):
             expected_args=['blah*', '/XF', '*.foo', '*.bar']
         )
 
+    def test_exclude_files(self):
+        return self._run_simple_test(
+            r'D:\source', r'E:\dest', files=['blah*'],
+            exclude_files=['*.foo', '*.bar'],
+            expected_args=['blah*', '/XF', '*.foo', '*.bar']
+        )
+
+    def test_exclude_dirs(self):
+        return self._run_simple_test(
+            r'D:\source', r'E:\dest', files=['blah*'],
+            exclude_dirs=['foo', 'bar'],
+            expected_args=['blah*', '/XD', 'foo', 'bar']
+        )
+
     @defer.inlineCallbacks
     def test_codes(self):
         # Codes that mean uneventful copies (including no copy at all).

master/buildbot/test/unit/test_steps_source_repo.py

@@ -112,14 +112,15 @@ def expectnoClobber(self):
             + 0,
         )
 
-    def expectRepoSync(self, which_fail=-1, breakatfail=False, syncoptions=["-c"], override_commands=[]):
+    def expectRepoSync(self, which_fail=-1, breakatfail=False, depth=0,
+                       syncoptions=["-c"], override_commands=[]):
         commands = [
             self.ExpectShell(
                 command=[
                     'bash', '-c', self.step._getCleanupCommand()]),
             self.ExpectShell(
                 command=['repo', 'init', '-u', 'git://myrepo.com/manifest.git',
-                         '-b', 'mb', '-m', 'mf'])
+                         '-b', 'mb', '-m', 'mf', '--depth', str(depth)])
         ] + override_commands + [
             self.ExpectShell(command=['repo', 'sync'] + syncoptions),
             self.ExpectShell(
@@ -137,6 +138,13 @@ def test_basic(self):
         self.expectRepoSync()
         return self.myRunStep(status_text=["update"])
 
+    def test_basic_depth(self):
+        """basic first time repo sync"""
+        self.mySetupStep(repoDownloads=None, depth=2)
+        self.expectClobber()
+        self.expectRepoSync(depth=2)
+        return self.myRunStep(status_text=["update"])
+
     def test_update(self):
         """basic second time repo sync"""
         self.mySetupStep()
@@ -387,7 +395,7 @@ def test_repo_download_manifest(self):
             + 0,
             self.ExpectShell(
                 command=['repo', 'init', '-u', 'git://myrepo.com/manifest.git',
-                         '-b', 'mb', '-m', 'mf'])
+                         '-b', 'mb', '-m', 'mf', '--depth', '0'])
             + 0,
             self.ExpectShell(
                 workdir='wkdir/.repo/manifests',

master/buildbot/test/unit/test_util_lru.py

@@ -252,7 +252,7 @@ def test_put_nonexistent_key(self):
         self.assertEqual(self.lru.get('p'), short('p'))
         self.lru.put('q', set(['new-q']))
         self.assertEqual(self.lru.get('p'), set(['PPP']))
-        self.assertEqual(self.lru.get('q'), set(['QQQ']))  # not updated
+        self.assertEqual(self.lru.get('q'), set(['new-q']))  # updated
 
 
 class AsyncLRUCacheTest(unittest.TestCase):

master/buildbot/util/lru.py

@@ -45,11 +45,12 @@ def __init__(self, miss_fn, max_size=50):
         self.miss_fn = miss_fn
 
     def put(self, key, value):
-        if key in self.cache:
-            self.cache[key] = value
-            self.weakrefs[key] = value
-        elif key in self.weakrefs:
-            self.weakrefs[key] = value
+        cached = key in self.cache or key in self.weakrefs
+        self.cache[key] = value
+        self.weakrefs[key] = value
+        self._ref_key(key)
+        if not cached:
+            self._purge()
 
     def get(self, key, **miss_fn_kwargs):
         try:

master/docs/developer/utils.rst

@@ -250,10 +250,9 @@ buildbot.util.lru
         :param key: key at which to place the value
         :param value: value to place there
 
-        Update the cache with the given key and value, but only if the key is
-        already in the cache.  The purpose of this method is to insert a new
-        value into the cache *without* invoking the miss_fn (e.g., to avoid
-        unnecessary overhead).
+        Add the given key and value into the cache.  The purpose of this
+        method is to insert a new value into the cache *without* invoking
+        the miss_fn (e.g., to avoid unnecessary overhead).
 
     .. py:method set_max_size(max_size)