Merge pull request #279 from twisted-infra/ubuntu-2004-migration

Update for Ubuntu 20.04 and make sure braid code works for a migration.
twisted-infra · Oct 29, 2021 · deffb8e · deffb8e
2 parents dd706ce + 910fa77
commit deffb8e
Show file tree

Hide file tree

Showing 14 changed files with 77 additions and 76 deletions.
diff --git a/braid/base.py b/braid/base.py
@@ -4,7 +4,7 @@
 
 from twisted.python.filepath import FilePath
 
-from braid import pypy, service, authbind, git, package, bazaar, postgres
+from braid import service, authbind, git, package, postgres
 
 
 __all__ = ['bootstrap', 'sshConfig']
@@ -28,7 +28,13 @@ def bootstrap():
     # libssl-dev is needed for installing pyOpenSSL for PyPy.
     package.install(['libssl-dev', 'libffi-dev'])
 
-    package.install(['python2.7', 'python2.7-dev', 'python-virtualenv'])
+    package.install(['python2.7-dev', 'pypy-dev' ])
+
+    # We don't have pip for python2.7 on Ubuntu 20.04
+    sudo('curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py')
+    sudo('python2 get-pip.py')
+    sudo('pip2 install virtualenv')
+
     # gcc and svn is needed for 'pip install'
     package.install(['gcc', 'subversion'])
     # For trac
@@ -37,10 +43,8 @@ def bootstrap():
     package.install(['equivs'])
     # For buildbot/codespeed
     package.install(['sqlite3'])
-    pypy.install()
     authbind.install()
     git.install()
-    bazaar.install()
     postgres.install()
 
     sshConfig()

diff --git a/braid/bazaar.py b/braid/bazaar.py
diff --git a/braid/postgres.py b/braid/postgres.py
@@ -4,10 +4,7 @@
 
 
 def install():
-    try:
-        package.install(['postgresql-9.1', 'postgresql-server-dev-9.1'])
-    except:
-        package.install(['postgresql-9.3', 'postgresql-server-dev-9.3'])
+    package.install(['postgresql-12'])
 
 
 def _runQuery(query, database=None):
@@ -108,6 +105,7 @@ def dumpToPath(database, dumpPath):
         '--blobs',
         '--no-owner',
         '--format', 'custom',
+        '--create',
         '--file', dumpPath,
         '--compress', '9',
         database,

diff --git a/braid/pypy.py b/braid/pypy.py
diff --git a/braid/service.py b/braid/service.py
@@ -3,7 +3,7 @@
 
 def _service(action, service, useSudo=True):
     cmd = sudo if useSudo else run
-    cmd('/usr/bin/service {} {}'.format(service, action))
+    cmd('/usr/bin/systemctl {} {}'.format(action, service))
 
 
 def start(service, useSudo=True):
@@ -19,10 +19,10 @@ def restart(service, useSudo=True):
 
 
 def enable(service, useSudo=True):
-    sudo('/usr/sbin/update-rc.d {} defaults'.format(service))
+    _service('enable', service, useSudo)
     _service('start', service, useSudo)
 
 
 def disable(service, useSudo=True):
+    _service('disable', service, useSudo)
     _service('stop', service, useSudo)
-    sudo('/usr/sbin/update-rc.d -f {} remove'.format(service))
diff --git a/braid/settings.py b/braid/settings.py
@@ -4,17 +4,27 @@
 
 dornkirk = 'dornkirk.twistedmatrix.com'
 buildbot = 'buildbot.twistedmatrix.com'
+twisted_vm = 'twisted-vm.centralus.cloudapp.azure.com'
 
 vagrant_address = '172.16.255.140'
 staging_address = '162.242.246.197'
 
 # For available options see http://docs.fabfile.org/en/latest/usage/env.html
 ENVIRONMENTS = {
     'production': {
+        'hosts': [twisted_vm],
+        'roledefs': {
+            'nameserver': [twisted_vm],
+            'buildbot': [twisted_vm],
+        },
+        'user': 'root',
+        'installPrivateData': True,
+    },
+    'dornkirk': {
         'hosts': [dornkirk],
         'roledefs': {
             'nameserver': [dornkirk],
-            'buildbot': [buildbot],
+            'buildbot': [dornkirk],
         },
         'user': 'root',
         'installPrivateData': True,

diff --git a/braid/venv.py b/braid/venv.py
@@ -7,7 +7,7 @@ class VirtualEnvironment(object):
     def __init__(self, user, location="~/virtualenv", python="python2.7"):
 
         if python == "pypy":
-            self._python = "~pypy/bin/pypy"
+            self._python = "/usr/bin/pypy"
         elif python == "python2.7":
             self._python = "/usr/bin/python2.7"
         else:
@@ -23,7 +23,7 @@ def create(self, site_packages=False):
         module.
         """
         with settings(user=self._user):
-            run(("/usr/bin/env python2 -m virtualenv --clear "
+            run(("/usr/bin/env python2 -m virtualenv "
                  "-p {} {} {}").format(self._python, self._location,
                                        '--system-site-packages' if site_packages else ''))
 

diff --git a/services/t-names/fabfile.py b/services/t-names/fabfile.py
@@ -3,7 +3,7 @@
 """
 import os
 
-from fabric.api import execute, put, run, settings
+from fabric.api import execute, put, run, sudo, settings
 
 from braid import authbind, cron
 from braid.twisted import service
@@ -22,7 +22,9 @@ def task_install(self):
         self.bootstrap()
 
         # Setup authbind
+        # Fix authbid, for now I just went with the generic setcap.
         authbind.allow(self.serviceUser, 53)
+        #sudo('setcap cap_net_bind_service=+ep /srv/t-names/virtualenv/bin/pypy')
 
         with settings(user=self.serviceUser):
             run('/bin/ln -nsf {}/start {}/start'.format(self.configDir, self.binDir))

diff --git a/services/t-names/start b/services/t-names/start
@@ -3,10 +3,13 @@
 Z=~/config/zones
 export PYTHONPATH=$PYTHONPATH:$Z
 
+# localhost:53 is used by local resolver.
+# Update t-names to remove the local resolver.
 authbind --deep ~/virtualenv/bin/twistd \
     --logfile ~/log/twistd.log \
     --pidfile ~/run/twistd.pid \
     dns \
+    -i 10.0.0.4 \
     --pyzone $Z/twistedmatrix.com \
     --pyzone $Z/divunal.com \
     --pyzone $Z/ynchrono.us \

diff --git a/services/t-names/zones/hosts.py b/services/t-names/zones/hosts.py
@@ -21,6 +21,7 @@
 dornkirk = '66.35.39.66'
 jeb = '162.242.246.197'
 buildbot = '104.130.225.105'
+azure_vm = '13.86.118.140'
 
 
 def nameservers(host, *addresses):

diff --git a/services/t-names/zones/twistedmatrix.com b/services/t-names/zones/twistedmatrix.com
@@ -1,3 +1,7 @@
+#
+# This is no longer used as it was moved to Gandi.
+# Here left for reference.
+#
 
 from twisted.names.authority import getSerial
 

diff --git a/services/t-names/zones/ynchrono.us b/services/t-names/zones/ynchrono.us
@@ -3,10 +3,10 @@ from twisted.names.authority import getSerial
 
 name = 'ynchrono.us'
 
-from hosts import dornkirk, ns2, addSubdomains
+from hosts import azure_vm, ns2, addSubdomains
 
 subs = {
-    dornkirk: ['ns1.', ''],
+    azure_vm: ['ns1.', ''],
     ns2: ['ns2.', ''],
     '104.130.225.186': ['chronology.'],
 }

diff --git a/services/trac/README.rst b/services/trac/README.rst
@@ -34,3 +34,14 @@ More information about how the post-commit hook works is available in the `trac-
 Use `fab -l` to check the available commands.
 
 Trac is available inside the Vagrant VM at http://172.16.255.140/trac
+
+
+GitHub Login
+------------
+
+First create a an OAuth Application or update the existing one at
+https://github.com/organizations/twisted/settings/applications
+
+Set/update the public client ID inside trac-env/conf/trac.ini
+
+Manually save the file as user trac  (mode 600) in secrets/secret.txt
diff --git a/services/trac/fabfile.py b/services/trac/fabfile.py
@@ -41,6 +41,14 @@ def task_install(self):
         postgres.createUser('trac')
         postgres.createDb('trac', 'trac')
 
+        # We don't have a braid rule for highscore,
+        # so I am just hacking this here to have a functional restore.
+        postgres.createUser('highscore')
+        postgres.grantRead('trac', 'highscore')
+        # amptrack has a dedicated braid, but is better to have it here
+        # so that dump and restore don't depend on it.
+        postgres.createUser('amptrac')
+        postgres.grantRead('trac', 'amptrac')
 
     def update(self):
         """
@@ -61,6 +69,12 @@ def update(self):
                 git+https://github.com/twisted-infra/twisted-trac-plugins.git
             """.split()))
 
+            # This is txacme v2 but is not yet released.
+            # Should be replaced on we have txacme v2.
+            # See https://github.com/twisted/txacme/pull/158
+            self.venv.install(
+                "--index=https://pypi.chevah.com/simple txacme==1.0.0.chevah4")
+
             run('mkdir -p ' + self.configDir)
             put(os.path.dirname(__file__) + '/*', self.configDir,
                 mirror_local_mode=True)
@@ -104,6 +118,13 @@ def task_dump(self, localfile, withAttachments=True):
         """
         Create a tarball containing all information not currently stored in
         version control and download it to the given C{localfile}.
+
+        C{localfile} is a path on your local system.
+
+        Keep it in sync with task_restore.
+
+        For it to work you need your SSH public key added to
+        /srv/trac/.ssh/authorized_keys
         """
         with settings(user=self.serviceUser):
             with utils.tempfile() as temp:
@@ -114,7 +135,8 @@ def task_dump(self, localfile, withAttachments=True):
                 }
 
                 if withAttachments is True:
-                    files['attachments'] = 'attachments'
+                    #files['attachments'] = 'attachments'
+                    files['trac-attachments'] = 'config/trac-env/files/attachments'
 
                 archive.dump(files, localfile)
 
@@ -123,6 +145,8 @@ def task_restore(self, localfile, restoreDb=True, withAttachments=True):
         """
         Restore all information not stored in version control from a tarball
         on the invoking users machine.
+
+        Keep it in sync with task_dump.
         """
         restoreDb = str(restoreDb).lower() in ('true', '1', 'yes', 'ok', 'y')
 
@@ -151,7 +175,8 @@ def task_restore(self, localfile, restoreDb=True, withAttachments=True):
                     }
 
                     if withAttachments is True:
-                        files['attachments'] = 'attachments'
+                        #files['attachments'] = 'attachments'
+                        files['trac-attachments'] = 'config/trac-env/files/attachments'
 
                     archive.restore(files, localfile)
                     if restoreDb: