[CDSK-815] Create a script which populates a database with random builds and user

.travis.yml

@@ -24,6 +24,7 @@ install:
   - if [ "$TEST_TARGET" != "javascript" ]; then pip install -v pyflakes; fi
   - if [ "$TEST_TARGET" != "javascript" ]; then pip install -v coveralls; fi
   - if [ "$TEST_TARGET" = "master" ]; then pip install service_identity; fi
+  - if [ "$TEST_TARGET" = "master" ]; then pip install -r master/requirements-dev.txt; fi
   - if [ "$TEST_TARGET" = "javascript" ]; then cd www/; fi
   - if [ "$TEST_TARGET" = "javascript" ]; then npm install; fi
 

master/buildbot/db/builds.py

@@ -13,8 +13,8 @@
 #
 # Copyright Buildbot Team Members
 from datetime import datetime, timedelta
+from twisted.internet import defer
 
-from sqlalchemy.orm import aliased
 from twisted.internet import reactor
 from buildbot.db import base
 from buildbot.util import epoch2datetime, datetime2epoch
@@ -276,6 +276,83 @@ def thd(conn):
             return [self._minimal_bdict(row, botmaster) for row in res.fetchall()]
         return self.db.pool.do(thd)
 
+    def createFullBuildObject(self, branch, revision, repository, project, reason, submitted_at,
+                              complete_at, buildername, slavepool, number, slavename, results, codebase):
+        """ This method creates a new build object with all required associated objects
+
+        :param branch: a string value with branch name (on this branch code was built)
+        :param revision: a string value with revision (on this revision code was built)
+        :param repository: a string value with path to repository
+        :param project: a string value with project name
+        :param reason: a string value described why build was executed
+        :param submitted_at: an integer value described when build was executed
+        :param complete_at: an integer value describe when build was completed or None when is still in progress
+        :param buildername: an string value with builder name, this name must exists in master.cfg
+        :param slavepool: a string value with slave pool name
+        :param number: an integer value with build number. Must be unique with build request id
+        :param slavename: a string value with slave name
+        :param results: an integer value with results status. See available options: master.buildbot.status.results
+        :param codebase: a string value with codebase of repository
+        :return: defer value
+        """
+        def thd(conn):
+            transaction = conn.begin()
+            try:
+                # Create sourcestampsets
+                r = conn.execute(self.db.model.sourcestampsets.insert(), dict())
+                sourcestampsset_id = r.inserted_primary_key[0]
+
+                # Create sourcestamps
+                conn.execute(self.db.model.sourcestamps.insert(), {
+                    'branch': branch,
+                    'revision': revision,
+                    'patchid': None,
+                    'repository': repository,
+                    'codebase': codebase,
+                    'project': project,
+                    'sourcestampsetid': sourcestampsset_id,
+                })
+
+                # Create buildsets
+                res = conn.execute(self.db.model.buildsets.insert(), {
+                    'reason': reason,
+                    'sourcestampsetid': sourcestampsset_id,
+                    'submitted_at': submitted_at,
+                    'complete': bool(complete_at),
+                    'complete_at': complete_at,
+                    'results': results,
+                })
+                buildset_id = res.inserted_primary_key[0]
+
+                # Create buildrequests
+                res = conn.execute(self.db.model.buildrequests.insert(), {
+                    'buildsetid': buildset_id,
+                    'buildername': buildername,
+                    'proiority': 50,
+                    'complete': bool(complete_at),
+                    'results': results,
+                    'submitted_at': submitted_at,
+                    'complete_at': complete_at,
+                    'slavepool': slavepool,
+                })
+                buildrequest_id = res.inserted_primary_key[0]
+
+                # Create builds
+                conn.execute(self.db.model.builds.insert(), {
+                    'number': number,
+                    'brid': buildrequest_id,
+                    'slavename': slavename,
+                    'start_time': submitted_at,
+                    'finish_time': complete_at,
+                })
+                transaction.commit()
+            except Exception as e:
+                print("Exception occurs during create new build", e)
+                transaction.rollback()
+                raise
+
+        return self.db.pool.do(thd)
+
     def _bdictFromRow(self, row):
         def mkdt(epoch):
             if epoch:

master/buildbot/db/users.py

@@ -14,6 +14,7 @@
 # Copyright Buildbot Team Members
 
 import sqlalchemy as sa
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.sql.expression import and_
 
 from buildbot.db import base
@@ -312,3 +313,24 @@ def thd(conn):
             return row.uid
         d = self.db.pool.do(thd)
         return d
+
+    def createUser(self, user):
+        """ This method creates users in a database.
+            :param user: user, if the user exists in the database, it will skip it
+            :type user: dictionary with 'identifier', 'bb_username' and 'bb_password' fields.
+
+            :return: defer
+        """
+        def thd(conn):
+            tbl = self.db.model.users
+            q = tbl.insert()
+            try:
+                conn.execute(q, user)
+                return True
+            except IntegrityError:
+                return False
+            except Exception as e:
+                print("An exception occurs during creating users", e)
+                raise e
+
+        return self.db.pool.do(thd)

master/buildbot/scripts/populatedatabase.py

@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+from __future__ import print_function
+
+import random
+import sys
+import traceback
+from time import time
+
+import datetime
+from twisted.internet import defer
+
+from buildbot import config as config_module
+from buildbot.db import connector
+from buildbot.master import BuildMaster
+from buildbot.util import in_reactor
+from buildbot.util import datetime2epoch
+from buildbot.status.results import COMPLETED_RESULTS
+
+
+MAX_UNIQUE_USER_COUNT = 5494
+
+
+@in_reactor
+@defer.inlineCallbacks
+def populate_database(config):
+    master = BuildMaster(config['baseDir'])
+    master.config = load_config(config, config['configFile'])
+    db = connector.DBConnector(master, basedir=config['baseDir'])
+    seed = int(time())
+    if config['seed']:
+        seed = int(config['seed'])
+    random.seed(seed)
+    if not config['quiet']:
+        print("Seed =", seed)
+
+    yield db.setup(check_version=False, verbose=not config['quiet'])
+    users = yield populate_user(db, int(config['users']), verbose=not config['quiet'])
+    yield populate_build(
+        db,
+        int(config['builds']),
+        master.config.builders,
+        master.config.projects,
+        users,
+        verbose=not config['quiet']
+    )
+
+
+def load_config(config, config_file_name='master.cfg'):
+    if not config['quiet']:
+        print("checking %s" % config_file_name)
+
+    try:
+        master_cfg = config_module.MasterConfig.loadConfig(
+            config['baseDir'],
+            config_file_name,
+        )
+    except config_module.ConfigErrors as e:
+        print("Errors loading configuration:")
+        for msg in e.errors:
+            print("  " + msg)
+        return
+    except Exception:
+        print("Errors loading configuration:")
+        traceback.print_exc(file=sys.stdout)
+        return
+
+    return master_cfg
+
+
+@defer.inlineCallbacks
+def populate_user(db, user_count, verbose=True):
+    """
+    This function create `user_count` number of random user in database
+    :param db: a handler to the DBConnection object
+    :param user_count: an integer value with number of new users
+    """
+    import names
+    from progress_bar import InitBar
+
+    if verbose:
+        print("Starting creating users")
+    if user_count > MAX_UNIQUE_USER_COUNT:
+        raise ValueError("Can not generate more than %d unique user" % MAX_UNIQUE_USER_COUNT)
+
+    users = []
+    created = 0
+    progress_bar = InitBar(size=user_count, stream=sys.stdout)
+
+    unique_identifier = set()
+    for ind in range(user_count):
+        # generate random identifier
+        identifier = names.get_first_name()
+        attempt = 0
+        while identifier in unique_identifier:
+            identifier = names.get_first_name()
+            attempt += 1
+            if attempt > user_count:
+                raise RuntimeError("Can not find unique name. Please choose small amount of records")
+
+        unique_identifier.add(identifier)
+        user = {
+            'identifier': identifier,
+            'bb_username': identifier,
+            'bb_password': 'pyflakes'
+        }
+        result = yield db.users.createUser(user)
+        if result:
+            created += 1
+        users.append(user)
+        if verbose:
+            progress_bar(ind+1)
+
+    if verbose:
+        print()
+        print("Created %d new users, %d skipped" % (created, user_count - created))
+
+    defer.returnValue(map(lambda x: x['identifier'], users))
+
+
+@defer.inlineCallbacks
+def populate_build(db, build_count, builders_list, projects, user_names, verbose=True):
+    """
+    :param db: a handler to the DBConnection object
+    :param build_count: an integer value with number of new builds
+    :param builders_list: a list of builders. The builder is a BuilderConfig object
+    :param projects: a list of a ProjectConfig objects
+    :param user_names: a list of an usernames (identifier) from the database
+    :param verbose: a boolean value indicate to print all information to std output
+    """
+    from progress_bar import InitBar
+
+    def handler(result, counter, *args):
+        result[counter] += 1
+
+    progress_bar = InitBar(size=build_count, stream=sys.stdout)
+
+    if verbose:
+        print("Starting creating builds")
+    res = {
+        'created': 0,
+        'skipped': 0,
+    }
+
+    for number in range(build_count):
+        builder = random.choice(builders_list)
+        codebases = random.choice(projects[builder.project].codebases)
+        codebase = random.choice(codebases.keys())
+        repository = codebases[codebase]
+        submitted_at = datetime2epoch(
+            datetime.datetime.now() + datetime.timedelta(seconds=random.randint(-3*60*60, -3*60*60))
+        )
+        complete_at = submitted_at + random.randint(60 * 60, 3 * 60 * 60)
+        build = {
+            'branch': repository['branch'],
+            'revision': "%032x" % random.getrandbits(160),  # Random sha-1 hash
+            'repository': repository['repository'],
+            'codebase': codebase,
+            'project': builder.project,
+            'reason': 'A build was forced by {username} {username}@localhost'.format(username=random.choice(user_names)),
+            'submitted_at': submitted_at,
+            'complete_at': complete_at,
+            'buildername': builder.name,
+            'slavepool': None,
+            'number': number,
+            'slavename': random.choice(builder.slavenames),
+            'results': random.choice(COMPLETED_RESULTS),
+        }
+        promise = db.builds.createFullBuildObject(**build)
+        promise.addCallback(lambda *args: handler(res, 'created'))
+        promise.addErrback(lambda *args: handler(res, 'skipped'))
+        yield promise
+
+        if verbose:
+            progress_bar(number+1)
+
+    if verbose:
+        print()
+        print("Created %d new builds, %d skipped" % (res['created'], res['skipped']))

master/buildbot/scripts/runner.py

@@ -22,13 +22,16 @@
 # pages and texinfo documentation.
 
 from twisted.python import usage, reflect
+import os
 import re
 import sys
 
 from buildbot.scripts import base
 
 # Note that the terms 'options' and 'config' are used interchangeably here - in
 # fact, they are interchanged several times.  Caveat legator.
+from buildbot.scripts.populatedatabase import MAX_UNIQUE_USER_COUNT
+
 
 def validate_master_option(master):
     """Validate master (-m, --master) command line option.
@@ -671,6 +674,33 @@ def postOptions(self):
                                        "or 'get'")
 
 
+class PopulateDatabaseOptions(base.SubcommandOptions):
+    subcommandFunction = "buildbot.scripts.populatedatabase.populate_database"
+    optFlags = [
+        ['quiet', 'q', "Don't display error messages or tracebacks"],
+    ]
+    optParameters = [
+        ["users", "u", 100,
+         "how many users should be created (max %d)" % MAX_UNIQUE_USER_COUNT],
+        ["builds", "b", 1000,
+         "how many builds should be created"],
+        ["seed", "s", None,
+         "seed for randomizer, default current timestamp"]
+    ]
+
+    def getSynopsis(self):
+        return "Usage:		buildbot populate-database [configFile]\n" + \
+         "		If not specified, './master.cfg' will be used as 'configFile'"
+
+    def parseArgs(self, *args):
+        if len(args) >= 1:
+            self['baseDir'] = os.path.dirname(args[0])
+            self['configFile'] = os.path.basename(args[0])
+        else:
+            self['baseDir'] = './'
+            self['configFile'] = 'master.cfg'
+
+
 class Options(usage.Options):
     synopsis = "Usage:    buildbot <command> [command options]"
 
@@ -704,9 +734,12 @@ class Options(usage.Options):
         ['checkconfig', None, CheckConfigOptions,
          "test the validity of a master.cfg config file"],
         ['user', None, UserOptions,
-         "Manage users in buildbot's database"]
+         "Manage users in buildbot's database"],
+        ['populate-database', None, PopulateDatabaseOptions,
+         'populate-database try to populate the database with sample randomized data'],
         ]
 
+
     def opt_version(self):
         import buildbot
         print "Buildbot version: %s" % buildbot.version