Add git repo/revision support to core and plugins.

When defined `git_url` and `git_revision` will **force**
the checkout to specified version before installation
of pulpcore and plugins.

fixes #6547

CHANGES/6547.feature

@@ -0,0 +1 @@
+Add git repo and revision to pulcore and plugin installer.

example.dev-config.yml

@@ -9,6 +9,8 @@ pulp_install_plugins:
   #   source_dir: "/home/vagrant/devel/pulp-2to3-migration"
   # pulp-ansible:
   #   source_dir: "/home/vagrant/devel/pulp_ansible"
+  #   # git_url: "https://github.com/pulp/pulp_ansible"  # optional git repo
+  #   # git_revision: "0.7.3"  # optional specific branch/tag/commit
   # pulp-certguard:
   #   source_dir: "/home/vagrant/devel/pulp-certguard"
   # pulp-container:
@@ -36,6 +38,10 @@ developer_user: "vagrant"
 developer_user_home: "/home/vagrant"
 pulp_source_dir: "/home/vagrant/devel/pulpcore"
 
+# Optional git specific branch
+# pulp_git_url: "https://github.com/pulp/pulpcore"
+# pulp_git_revision: "3.11"  # branch/tag/commit
+
 # # Uncomment to prevent https configuration
 # pulp_webserver_disable_https: true
 

molecule/source-dynamic/group_vars/all

@@ -1,6 +1,8 @@
 ---
 pulp_default_admin_password: password
 pulp_source_dir: '/var/lib/pulp/devel/pulpcore'
+pulp_git_url: "https://github.com/pulp/pulpcore"
+pulp_git_revision: "master"
 pulp_install_plugins:
   # galaxy-ng:
   #   source_dir: "/var/lib/pulp/devel/galaxy_ng"
@@ -15,6 +17,8 @@ pulp_install_plugins:
   # pulp-deb:
   #   source_dir: "/var/lib/pulp/devel/pulp_deb"
   pulp-file:
+    git_url: "https://github.com/pulp/pulp_file"
+    git_revision: "master"
     source_dir: "/var/lib/pulp/devel/pulp_file"
   # pulp-gem:
   #   source_dir: "/var/lib/pulp/devel/pulp_gem"
@@ -25,6 +29,8 @@ pulp_install_plugins:
   # pulp-python:
   #   source_dir: "/var/lib/pulp/devel/pulp_python"
   pulp-rpm:
+    git_url: "https://github.com/pulp/pulp_rpm"
+    git_revision: "master"
     source_dir: "/var/lib/pulp/devel/pulp_rpm"
 developer_user_home: /var/lib/pulp
 developer_user: pulp

molecule/source-static/group_vars/all

@@ -1,6 +1,8 @@
 ---
 pulp_default_admin_password: password
 pulp_source_dir: '/var/lib/pulp/devel/pulpcore'
+pulp_git_url: "https://github.com/pulp/pulpcore"
+pulp_git_revision: "master"
 pulp_api_bind: "unix:/var/run/pulpcore-api/pulpcore-api.sock"
 pulp_content_bind: "unix:/var/run/pulpcore-content/pulpcore-content.sock"
 pulp_install_plugins:
@@ -17,6 +19,8 @@ pulp_install_plugins:
   # pulp-deb:
   #   source_dir: "/var/lib/pulp/devel/pulp_deb"
   pulp-file:
+    git_url: "https://github.com/pulp/pulp_file"
+    git_revision: "master"
     source_dir: "/var/lib/pulp/devel/pulp_file"
   # pulp-gem:
   #   source_dir: "/var/lib/pulp/devel/pulp_gem"
@@ -27,6 +31,8 @@ pulp_install_plugins:
   # pulp-python:
   #   source_dir: "/var/lib/pulp/devel/pulp_python"
   pulp-rpm:
+    git_url: "https://github.com/pulp/pulp_rpm"
+    git_revision: "master"
     source_dir: "/var/lib/pulp/devel/pulp_rpm"
 developer_user_home: /var/lib/pulp
 developer_user: pulp

molecule/source-static/host_vars/debian-10

@@ -13,6 +13,7 @@ pulp_install_plugins:
   # pulp-deb:
   #   source_dir: "/var/lib/pulp/devel/pulp_deb"
   pulp-file:
+    git_url: "https://github.com/pulp/pulp_file"
     source_dir: "/var/lib/pulp/devel/pulp_file"
   # pulp-gem:
   #   source_dir: "/var/lib/pulp/devel/pulp_gem"

molecule/source-static/prepare.yml

@@ -14,21 +14,6 @@
         home: '{{ developer_user_home }}'
       when: developer_user is defined
 
-    - name: Clone pulp repository
-      git:
-        repo: 'https://github.com/pulp/pulpcore.git'
-        dest: '{{ pulp_source_dir }}'
-        version: master
-        update: yes
-
-    - name: Clone pulp plugin repositories
-      git:
-        repo: '{{ item.value.git_repo | default( "https://github.com/pulp/" + item.key | replace("-", "_") ) }}'
-        dest: '{{ item.value.source_dir }}'
-        version: master
-        update: yes
-      with_dict: '{{ pulp_install_plugins }}'
-
     - name: Make devel directory world-writable
       file:
         path: /var/lib/pulp/

molecule/source-upgrade/group_vars/all

@@ -2,15 +2,21 @@
 __pulp_database_config_run_once: false
 pulp_default_admin_password: password
 pulp_source_dir: '/var/lib/pulp/devel/pulpcore'
+pulp_git_url: "https://github.com/pulp/pulpcore"
+pulp_git_revision: "master"
 pulp_upgrade: true
 # Needed to determine whether or not pulpcore was actually upgraded, to
 # trigger the handler.
 pulp_pip_editable: false
 pulp_install_plugins:
   pulp-file:
+    git_url: "https://github.com/pulp/pulp_file"
+    git_revision: "master"
     upgrade: true
     source_dir: "/var/lib/pulp/devel/pulp_file"
   pulp-rpm:
+    git_url: "https://github.com/pulp/pulp_rpm"
+    git_revision: "master"
     upgrade: true
     source_dir: "/var/lib/pulp/devel/pulp_rpm"
 developer_user_home: /var/lib/pulp

molecule/source-upgrade/host_vars/debian-10

@@ -2,6 +2,7 @@ ansible_python_interpreter: /usr/bin/python3
 pulp_install_plugins:
   pulp-file:
     upgrade: true
+    git_url: "https://github.com/pulp/pulp_file"
     source_dir: "/var/lib/pulp/devel/pulp_file"
   # pulp-rpm:
   #   upgrade: true

roles/pulp_common/README.md

@@ -49,6 +49,10 @@ Role Variables
         prereq_role: "pulp.pulp_four_role" # role to run immediately before the venv is created
       pulp-five:
         collectstatic: false
+      pulp-six:
+        source_dir: "/path/to/source/code/"
+        git_url: "https://github..."  # Optional. URL to the git repo from where plugin will be pulled.
+        git_revision: "v3.1.1"   # Optional. The specific git branch/tag/commit to be cheked out.
     ```
 * `pulp_cache_dir`: Location of Pulp cache. Defaults to "/var/lib/pulp/tmp".
 * `pulp_config_dir`: Directory which will contain Pulp configuration files.
@@ -60,6 +64,15 @@ Role Variables
 * `pulp_source_dir`: Optional. Absolute path to pulpcore source code. If
   present, pulpcore will be installed from source in editable mode. Also accepts
   a pip VCS URL, to (for example) install the master branch.
+* `pulp_git_url`: Optional. URL to the git repository from where pulpcore will be checked out if
+  doesn't exists already on `source_dir`.
+  > **WARNING** when `pulp_git_url` is defined this role will clone the repo if doesn't already
+  > exist in the `pulp_source_dir` location and also checkout/update to specified `pulp_git_revision` if
+  > provided. In the case of existing local repo with unstaged changes the update will NOT be forced.
+  > For development purposes the recommendation is to NOT provide `pulp_git_url` and manage local
+  > branches manually.
+* `pulp_git_revision`: Optional. The specific git branch/tag/commit to be cheked out
+  if git_url is provided.
 * `pulp_user`: User that owns and runs Pulp. Defaults to "pulp".
 * `pulp_user_id`: Integer value of uid for the `pulp_user`. Defaults to nothing and uid is assigned
   by the system.

roles/pulp_common/tasks/install_pip.yml

@@ -211,6 +211,18 @@
         - Collect static content
         - Restart all Pulp services
 
+    - name: Clone or Checkout pulpcore source git repo
+      git:
+        repo: '{{ pulp_git_url }}'
+        dest: '{{ pulp_source_dir }}'
+        clone: yes
+        force: no
+        version: '{{ pulp_git_revision | default("master") }}'
+        update: yes
+      when: pulp_git_url is defined and pulp_source_dir is defined
+      tags:
+        - molecule-idempotence-notest
+
     - name: Install pulpcore package from source
       pip:
         name: '{{ pulp_source_dir }}'
@@ -286,6 +298,19 @@
         - Collect static content
         - Restart all Pulp services
 
+    - name: Clone or Checkout pulp plugins source git repos
+      git:
+        repo: '{{ item.value.git_url }}'
+        dest: '{{ item.value.source_dir }}'
+        clone: yes
+        force: no
+        version: '{{ item.value.git_revision | default("master") }}'
+        update: yes
+      with_dict: '{{ pulp_install_plugins_normalized }}'
+      when: item.value.git_url is defined and item.value.source_dir is defined
+      tags:
+        - molecule-idempotence-notest
+
     - name: Install Pulp plugins from source
       pip:
         name: '{{ item.value.source_dir }}'