Add more utility functions for identifiers

master/buildbot/db/buildslaves.py

@@ -16,7 +16,7 @@
 import sqlalchemy as sa
 
 from buildbot.db import base
-from buildbot.util import typechecks
+from buildbot.util import identifiers
 from twisted.internet import defer
 
 
@@ -26,7 +26,7 @@ class BuildslavesConnectorComponent(base.DBConnectorComponent):
     def findBuildslaveId(self, name):
         tbl = self.db.model.buildslaves
         # callers should verify this and give good user error messages
-        assert typechecks.isIdentifier(50, name)
+        assert identifiers.isIdentifier(50, name)
         return self.findSomethingId(
             tbl=tbl,
             whereclause=(tbl.c.name == name),

master/buildbot/test/unit/test_util_identifiers.py

@@ -0,0 +1,100 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+from buildbot.util import identifiers
+from twisted.python import log
+from twisted.trial import unittest
+
+
+class Tests(unittest.TestCase):
+
+    def test_isIdentifier(self):
+        good=[
+            u"linux", u"Linux", u"abc123", u"a" * 50,
+        ]
+        for g in good:
+            log.msg('expect %r to be good' % (g,))
+            self.assertTrue(identifiers.isIdentifier(50, g))
+        bad=[
+            None, u'', 'linux', u'a/b', u'\N{SNOWMAN}', u"a.b.c.d",
+            u"a-b_c.d9", 'spaces not allowed', u"a" * 51,
+            u"123 no initial digits",
+        ]
+        for b in bad:
+            log.msg('expect %r to be bad' % (b,))
+            self.assertFalse(identifiers.isIdentifier(50, b))
+
+    def assertEqualUnicode(self, got, exp):
+        self.failUnless(isinstance(exp, unicode))
+        self.assertEqual(got, exp)
+
+    def test_forceIdentifier_already_is(self):
+        self.assertEqualUnicode(
+                identifiers.forceIdentifier(10, u'abc'),
+                u'abc')
+
+    def test_forceIdentifier_ascii(self):
+        self.assertEqualUnicode(
+                identifiers.forceIdentifier(10, 'abc'),
+                u'abc')
+
+    def test_forceIdentifier_too_long(self):
+        self.assertEqualUnicode(
+                identifiers.forceIdentifier(10, 'abcdefghijKL'),
+                u'abcdefghij')
+
+    def test_forceIdentifier_invalid_chars(self):
+        self.assertEqualUnicode(
+                identifiers.forceIdentifier(100, 'my log.html'),
+                u'my_log_html')
+
+    def test_forceIdentifier_leading_digit(self):
+        self.assertEqualUnicode(
+                identifiers.forceIdentifier(100, '9 pictures of cats.html'),
+                u'__pictures_of_cats_html')
+
+    def test_incrementIdentifier_simple(self):
+        self.assertEqualUnicode(
+                identifiers.incrementIdentifier(100, u'aaa'),
+                u'aaa_2')
+
+    def test_incrementIdentifier_simple_way_too_long(self):
+        self.assertEqualUnicode(
+                identifiers.incrementIdentifier(3, u'aaa'),
+                u'a_2')
+
+    def test_incrementIdentifier_simple_too_long(self):
+        self.assertEqualUnicode(
+                identifiers.incrementIdentifier(4, u'aaa'),
+                u'aa_2')
+
+    def test_incrementIdentifier_single_digit(self):
+        self.assertEqualUnicode(
+                identifiers.incrementIdentifier(100, u'aaa_2'),
+                u'aaa_3')
+
+    def test_incrementIdentifier_add_digits(self):
+        self.assertEqualUnicode(
+                identifiers.incrementIdentifier(100, u'aaa_99'),
+                u'aaa_100')
+
+    def test_incrementIdentifier_add_digits_too_long(self):
+        self.assertEqualUnicode(
+                identifiers.incrementIdentifier(6, u'aaa_99'),
+                u'aa_100')
+
+    def test_incrementIdentifier_add_digits_out_of_space(self):
+        self.assertRaises(ValueError, lambda: \
+                identifiers.incrementIdentifier(6, u'_99999'))

master/buildbot/util/identifiers.py

@@ -0,0 +1,61 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import re
+from buildbot import util
+
+ident_re = re.compile('^[a-zA-Z_-][a-zA-Z0-9_-]*$')
+initial_re = re.compile('^[^a-zA-Z_-]')
+subsequent_re = re.compile('[^a-zA-Z-1-9_-]')
+trailing_digits_re = re.compile('_([0-9]+)$')
+
+
+def isIdentifier(maxLength, object):
+    if not isinstance(object, unicode):
+        return False
+    elif not ident_re.match(object):
+        return False
+    elif not 0 < len(object) <= maxLength:
+        return False
+    return True
+
+
+def forceIdentifier(maxLength, str):
+    if not isinstance(str, basestring):
+        raise TypeError("%r cannot be coerced to an identifier" % (str,))
+
+    # usually ascii2unicode can handle it
+    str = util.ascii2unicode(str)
+    if isIdentifier(maxLength, str):
+        return str
+
+    # trim to length and substitute out invalid characters
+    str = str[:maxLength]
+    str = initial_re.sub('_', str)
+    str = subsequent_re.subn('_', str)[0]
+    return str
+
+
+def incrementIdentifier(maxLength, ident):
+    num = 1
+    mo = trailing_digits_re.search(ident)
+    if mo:
+        ident = ident[:mo.start(1) - 1]
+        num = int(mo.group(1))
+    num = '_%d' % (num + 1)
+    if len(num) > maxLength:
+        raise ValueError("cannot generate a larger identifier")
+    ident = ident[:maxLength - len(num)] + num
+    return ident

master/docs/developer/db.rst

@@ -57,7 +57,7 @@ Identifier
 
 An "identifier" is a nonempty unicode string of limited length, containing only ASCII alphanumeric characters along with ``-`` (dash) and ``_`` (underscore), and not beginning with a digit
 Wherever an identifier is used, the documentation will give the maximum length in characters.
-The function :py:func:`buildbot.util.typechecks.isIdentifier` is useful to verify a well-formed identifier.
+The function :py:func:`buildbot.util.identifiers.isIdentifier` is useful to verify a well-formed identifier.
 
 buildrequests
 ~~~~~~~~~~~~~

master/docs/developer/utils.rst

@@ -663,12 +663,12 @@ This module is a drop-in replacement for the stdlib ``pickle`` or ``cPickle`` mo
 It adds the ability to load pickles that reference classes that have since been removed from Buildbot.
 It should be used whenever pickles from Buildbot-0.8.x and earlier are loaded.
 
-buildbot.util.typechecks
-~~~~~~~~~~~~~~~~~~~~~~~~
+buildbot.util.identifiers
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. py:module:: buildbot.util.typechecks
+.. py:module:: buildbot.util.identifiers
 
-This module makes it easy to check argument types.
+This module makes it easy to manipulate identifiers.
 
 .. py:function:: isIdentifier(maxLength, object)
 
@@ -678,6 +678,26 @@ This module makes it easy to check argument types.
 
     Is object a :ref:`identifier <type-identifier>`?
 
+.. py:function:: forceIdentifier(maxLength, str)
+
+    :param maxLength: maximum length of the identifier
+    :param str: string to coerce to an identifier
+    :returns: identifer of maximum length ``maxLength``
+
+    Coerce a string (assuming ASCII for bytestrings) into an identifier.
+    This method will replace any invalid characters with ``_`` and truncate to the given length.
+
+.. py:function:: incrementIdentifier(maxLength, str)
+
+    :param maxLength: maximum length of the identifier
+    :param str: identifier to increment
+    :returns: identifer of maximum length ``maxLength``
+    :raises: ValueError if no suitable identifier can be constructed
+
+    "Increment" an identifier by adding a numeric suffix, while keeping the total length limited.
+    This is useful when selecting a unique identifier for an object.
+    Maximum-length identifiers like ``_999999`` cannot be incremented and will raise :py:exc:`ValueError`.
+
 buildbot.util.lineboundaries
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~