Switch docs builders to OSCI env

ci/docs-builder.py

@@ -3,18 +3,19 @@
 import argparse
 import subprocess
 import os
-from shutil import copyfile
+from shutil import copyfile, rmtree
+import tempfile
 
 from lib import builder, promote
 from lib.builder import WORKING_DIR
 
 
 LATEST = '2.14'
 
-USERNAME = '57600fb10c1e664383000229'
-HOSTNAME = 'docs-pulp.rhcloud.com'
+USERNAME = 'doc_builder'
+HOSTNAME = '8.43.85.236'
 
-SITE_ROOT = '~/app-root/repo/diy/'
+SITE_ROOT = '/var/www/docs.pulpproject.org/'
 
 # dict of {git repo: [list of package dirs containing setup.py]} that need to be installed
 # for apidoc generation to work; only used for pulp 3+
@@ -23,6 +24,30 @@
 }
 
 
+def make_directory_with_rsync(remote_paths_list):
+    """
+    Ensure the remote directory path exists
+
+    :param remote_paths_list: The list of parameters. e.g. ['en', 'latest'] to be en/latest on the
+        remote.
+    :type remote_paths_list: a list of strings, with each string representing a directory.
+    """
+    try:
+        tempdir_path = tempfile.mkdtemp()
+        cwd = os.getcwd()
+        os.chdir(tempdir_path)
+        os.makedirs(os.sep.join(remote_paths_list))
+        remote_path_arg = '%s@%s:%s%s' % (USERNAME, HOSTNAME, SITE_ROOT, remote_paths_list[0])
+        local_path_arg = tempdir_path + os.sep + remote_paths_list[0] + os.sep
+        rsync_command = ['rsync', '-avzh', local_path_arg, remote_path_arg]
+        exit_code = subprocess.call(rsync_command)
+        if exit_code != 0:
+            raise RuntimeError('An error occurred while creating remote directories.')
+    finally:
+        rmtree(tempdir_path)
+        os.chdir(cwd)
+
+
 def main():
     # Parse the args
     parser = argparse.ArgumentParser()
@@ -122,66 +147,46 @@ def main():
         local_path_arg = os.sep.join([docs_directory, '_build', 'html']) + os.sep
         remote_path_arg = '%s@%s:%s' % (USERNAME, HOSTNAME, SITE_ROOT)
         rsync_command = ['rsync', '-avzh', '--delete', '--exclude', 'en',
-                         local_path_arg, remote_path_arg]
+                         '--omit-dir-times', local_path_arg, remote_path_arg]
         exit_code = subprocess.call(rsync_command, cwd=docs_directory)
         if exit_code != 0:
-            raise RuntimeError('An error occurred while pushing latest docs to OpenShift.')
+            raise RuntimeError('An error occurred while pushing latest docs.')
+
+        # Also publish to the /en/latest/ directory
+        make_directory_with_rsync(['en', 'latest'])
+        local_path_arg = os.sep.join([docs_directory, '_build', 'html']) + os.sep
+        remote_path_arg = '%s@%s:%sen/latest/' % (USERNAME, HOSTNAME, SITE_ROOT)
+        rsync_command = ['rsync', '-avzh', '--delete', local_path_arg, remote_path_arg]
+        exit_code = subprocess.call(rsync_command, cwd=docs_directory)
+        if exit_code != 0:
+            raise RuntimeError("An error occurred while pushing the 'latest' directory.")
 
     # rsync the nightly "master" docs to an unversioned "nightly" dir for
     # easy linking to in-development docs: /en/nightly/
     if build_type == 'nightly' and opts.release == 'master':
         local_path_arg = os.sep.join([docs_directory, '_build', 'html']) + os.sep
         remote_path_arg = '%s@%s:%sen/%s/' % (USERNAME, HOSTNAME, SITE_ROOT, build_type)
-        path_option_arg = 'mkdir -p %sen/%s/ && rsync' % (SITE_ROOT, build_type)
-        rsync_command = ['rsync', '-avzh', '--rsync-path', path_option_arg, '--delete',
-                         local_path_arg, remote_path_arg]
+        make_directory_with_rsync(['en', build_type])
+        rsync_command = ['rsync', '-avzh', '--delete', local_path_arg, remote_path_arg]
         exit_code = subprocess.call(rsync_command, cwd=docs_directory)
         if exit_code != 0:
-            raise RuntimeError('An error occurred while pushing nightly docs to OpenShift.')
+            raise RuntimeError('An error occurred while pushing nightly docs.')
 
-    # rsync the docs to OpenShift
+    # rsync the docs
     local_path_arg = os.sep.join([docs_directory, '_build', 'html']) + os.sep
     remote_path_arg = '%s@%s:%sen/%s/' % (USERNAME, HOSTNAME, SITE_ROOT, x_y_version)
     if build_type != 'ga':
         remote_path_arg += build_type + '/'
-        path_option_arg = 'mkdir -p %sen/%s/%s/ && rsync' % (SITE_ROOT, x_y_version, build_type)
-        rsync_command = ['rsync', '-avzh', '--rsync-path', path_option_arg, '--delete',
-                         local_path_arg, remote_path_arg]
+
+        make_directory_with_rsync(['en', x_y_version, build_type])
+        rsync_command = ['rsync', '-avzh', '--delete', local_path_arg, remote_path_arg]
     else:
-        path_option_arg = 'mkdir -p %sen/%s/ && rsync' % (SITE_ROOT, x_y_version)
-        rsync_command = ['rsync', '-avzh', '--rsync-path', path_option_arg, '--delete',
-                         '--exclude', 'nightly', '--exclude', 'testing',
-                         local_path_arg, remote_path_arg]
+        make_directory_with_rsync(['en', x_y_version])
+        rsync_command = ['rsync', '-avzh', '--delete', '--exclude', 'nightly', '--exclude',
+                         'testing', local_path_arg, remote_path_arg]
     exit_code = subprocess.call(rsync_command, cwd=docs_directory)
     if exit_code != 0:
-        raise RuntimeError('An error occurred while pushing docs to OpenShift.')
-
-    # rsync the robots.txt to OpenShift
-    local_path_arg = 'docs/robots.txt'
-    remote_path_arg = '%s@%s:%s' % (USERNAME, HOSTNAME, SITE_ROOT)
-    scp_command = ['scp', local_path_arg, remote_path_arg]
-    exit_code = subprocess.call(scp_command)
-    if exit_code != 0:
-        raise RuntimeError('An error occurred while pushing robots.txt to OpenShift.')
-
-    # rsync the testrubyserver.rb to OpenShift
-    local_path_arg = 'docs/testrubyserver.rb'
-    remote_path_arg = '%s@%s:%s' % (USERNAME, HOSTNAME, SITE_ROOT)
-    scp_command = ['scp', local_path_arg, remote_path_arg]
-    exit_code = subprocess.call(scp_command)
-    if exit_code != 0:
-        raise RuntimeError('An error occurred while pushing testrubyserver.rb to OpenShift.')
-
-    # add symlink for latest
-    symlink_cmd = [
-        'ssh',
-        '%s@%s' % (USERNAME, HOSTNAME),
-        'ln -sfn %sen/%s %sen/latest' % (SITE_ROOT, LATEST, SITE_ROOT)
-    ]
-    exit_code = subprocess.call(symlink_cmd)
-    if exit_code != 0:
-        raise RuntimeError("An error occurred while creating the 'latest' symlink "
-                           "testrubyserver.rb to OpenShift.")
+        raise RuntimeError('An error occurred while pushing docs.')
 
 
 if __name__ == "__main__":

ci/jjb/jobs/docs.yaml

@@ -21,7 +21,7 @@
         - ssh-agent-credentials:
             users:
                 - '044c0620-d67e-4172-9814-dc49e443e7b6'
-                - '2f7a0c14-d520-40d3-b9b5-ebd8bb780d03'
+                - '31be896c-3325-4158-9ed6-8d0c2716342a'
         - credentials-binding:
             - zip-file:
                 credential-id: 9051da21-c8af-49bd-a0ac-c1dd94a6d216
@@ -44,7 +44,7 @@
 
             # Add github.com as a known host
             echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" >> /home/jenkins/.ssh/known_hosts
-            echo "docs-pulp.rhcloud.com,52.1.83.65 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAwUPkUQ84FKIWhOxy5RBBuR9gbrov2epARmFmaxD7NFRayobnDvl8GaBTbH1kxaZ/tYQeIqVE1assL74KArMQBzz6rj0FEWf0zrXxAY55EGswmWEEdqlYu1LbIxDCG6opqkiq6ocxjea9K3XYq+2aYoAvI3sshSImTYZP1glFhrh3QUsNJHOfDboTLJFNSdLjzXubRMa4eEx7s9pe9kwBOaLNIiVeGzUWg5+KaykSg2UMB3aG127t8kX+OhDYceVR42ehQJ0MjQGlGoNtldxGrlX8NjxUqvJAo6pqNqRK8Cps7/x/m0GPXWAgSZymhurXmj1o2LP5nKLtVzMPvwMb0w==" >> /home/jenkins/.ssh/known_hosts
+            echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> /home/jenkins/.ssh/known_hosts
 
             chmod 644 /home/jenkins/.ssh/known_hosts
 
@@ -96,7 +96,7 @@
         - ssh-agent-credentials:
             users:
                 - '044c0620-d67e-4172-9814-dc49e443e7b6'
-                - '2f7a0c14-d520-40d3-b9b5-ebd8bb780d03'
+                - '31be896c-3325-4158-9ed6-8d0c2716342a'
         - credentials-binding:
             - zip-file:
                 credential-id: 9051da21-c8af-49bd-a0ac-c1dd94a6d216
@@ -119,7 +119,7 @@
 
             # Add github.com as a known host
             echo "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" >> /home/jenkins/.ssh/known_hosts
-            echo "docs-pulp.rhcloud.com,52.1.83.65 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAwUPkUQ84FKIWhOxy5RBBuR9gbrov2epARmFmaxD7NFRayobnDvl8GaBTbH1kxaZ/tYQeIqVE1assL74KArMQBzz6rj0FEWf0zrXxAY55EGswmWEEdqlYu1LbIxDCG6opqkiq6ocxjea9K3XYq+2aYoAvI3sshSImTYZP1glFhrh3QUsNJHOfDboTLJFNSdLjzXubRMa4eEx7s9pe9kwBOaLNIiVeGzUWg5+KaykSg2UMB3aG127t8kX+OhDYceVR42ehQJ0MjQGlGoNtldxGrlX8NjxUqvJAo6pqNqRK8Cps7/x/m0GPXWAgSZymhurXmj1o2LP5nKLtVzMPvwMb0w==" >> /home/jenkins/.ssh/known_hosts
+            echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> /home/jenkins/.ssh/known_hosts
 
             chmod 644 /home/jenkins/.ssh/known_hosts