[v2.5.x release branch] CI : Clean up config file, enable for tags, and update output directories

.gitlab-ci.yml

@@ -8,8 +8,8 @@ stages:
 
 variables:
   GITHUB_REPO_API_URL: "https://api.github.com/repos/ipbus/ipbus-software"
-  BUILD_OUTPUT_ROOT_DIR: "/afs/cern.ch/user/i/ipbusci/public/sw-gitlab-ci/commits"
-  BUILD_OUTPUT_ROOT_URL: "http://www.cern.ch/ipbus/sw/ci/commits"
+  OUTPUT_ROOT_DIR: "/eos/user/i/ipbusci/sw-gitlab-ci"
+  OUTPUT_ROOT_URL: "http://www.cern.ch/ipbus/sw/ci"
 
 
 
@@ -57,7 +57,6 @@ doxygen_job:
   tags:
     - docker
   except:
-    - tags
     - triggers
   before_script:
     - sudo yum -y install graphviz
@@ -68,21 +67,19 @@ doxygen_job:
     - cd doxygen-1.8.6
     - ./configure --prefix /usr --docdir /usr/share/doc/doxygen-1.8.6 && make
     - sudo make MAN1DIR=share/man/man1 install
-    - cd ..
-    - mkdir -p ~/.ssh
-    - echo "HOST lxplus" > ~/.ssh/config
-    - echo "  StrictHostKeyChecking no" >> ~/.ssh/config
-    - echo "  GSSAPITrustDns yes" >> ~/.ssh/config
-    - echo "  GSSAPIAuthentication yes" >> ~/.ssh/config
-    - echo "  GSSAPIDelegateCredentials yes" >> ~/.ssh/config
+    - mkdir -p ~/.ssh  &&  cp ${CI_PROJECT_DIR}/ci/ssh_config ~/.ssh/config
+    - export OUTPUT_PIPELINE_DIR=${OUTPUT_ROOT_DIR}/commits/${CI_COMMIT_TAG:-${CI_COMMIT_SHA}}/pipeline${CI_PIPELINE_ID}
+    - if [ -n "${CI_COMMIT_TAG}" ]; then export OUTPUT_PIPELINE_DIR=${OUTPUT_PIPELINE_DIR/commits/tags} ; fi
   script:
     - env | grep -v PASSWORD | grep -v TOKEN
-    - ./scripts/doxygen/api_uhal.sh "(commit ${CI_COMMIT_SHA})"
+    - cd ${CI_PROJECT_DIR}
+    - ./scripts/doxygen/api_uhal.sh "(${CI_COMMIT_TAG:-commit ${CI_COMMIT_SHA}})"
     - ls -al /tmp
     - echo "${KRB_PASSWORD}" | kinit ${KRB_USERNAME}@CERN.CH
     - klist
-    - ssh ${KRB_USERNAME}@lxplus "mkdir -p ${BUILD_OUTPUT_ROOT_DIR}/${CI_PIPELINE_ID}_${CI_BUILD_REF}" 
-    - scp -r /tmp/api_uhal ${KRB_USERNAME}@lxplus:${BUILD_OUTPUT_ROOT_DIR}/${CI_PIPELINE_ID}_${CI_BUILD_REF}/
+    - ssh ${KRB_USERNAME}@lxplus "mkdir -p ${OUTPUT_PIPELINE_DIR}" 
+    - ssh ${KRB_USERNAME}@lxplus "ln -sfn ${OUTPUT_PIPELINE_DIR} `dirname ${OUTPUT_PIPELINE_DIR}`/latest" 
+    - rsync -av /tmp/api_uhal ${KRB_USERNAME}@lxplus:${OUTPUT_PIPELINE_DIR}/
   artifacts:
     untracked: true
     when: always
@@ -91,83 +88,65 @@ doxygen_job:
 
 
 
-build:centos7:
+
+.job_template: &build_job
   stage: build
-  image: tswilliams/ipbus-sw-dev-cc7:latest
   tags:
     - docker
   except:
-    - tags
     - triggers
   before_script:
-    - sudo yum -y install boost-devel
-    - sudo yum -y install createrepo krb5-workstation openssh-clients
-    - mkdir -p ~/.ssh
-    - echo "HOST lxplus" > ~/.ssh/config
-    - echo "  StrictHostKeyChecking no" >> ~/.ssh/config
-    - echo "  GSSAPITrustDns yes" >> ~/.ssh/config
-    - echo "  GSSAPIAuthentication yes" >> ~/.ssh/config
-    - echo "  GSSAPIDelegateCredentials yes" >> ~/.ssh/config
+    - sudo yum -y install createrepo krb5-workstation openssh-clients ${ADDITIONAL_RPMS_TO_INSTALL}
+    - mkdir -p ~/.ssh  &&  cp ci/ssh_config ~/.ssh/config
+    - export OUTPUT_PIPELINE_DIR=${OUTPUT_ROOT_DIR}/commits/${CI_COMMIT_TAG:-${CI_COMMIT_SHA}}/pipeline${CI_PIPELINE_ID}
+    - if [ -n "${CI_COMMIT_TAG}" ]; then export OUTPUT_PIPELINE_DIR=${OUTPUT_PIPELINE_DIR/commits/tags} ; fi
   script:
     - env | grep -v PASSWORD | grep -v TOKEN
     - make -k Set=all
     - make -k Set=all rpm
-    - mkdir -p cc7_rpms
-    - cp `find . -iname "*.rpm"` cc7_rpms
-    - cp yumgroups-centos7.xml cc7_rpms/
-    - ls cc7_rpms
-    - createrepo -vg yumgroups-centos7.xml cc7_rpms/
+    - mkdir -p yumrepo
+    - cp `find . -iname "*.rpm"` yumrepo
+    - cp ${YUMGROUPS_FILE} yumrepo/yumgroups.xml
+    - ls yumrepo
+    - createrepo -vg yumgroups.xml yumrepo
     - echo "${KRB_PASSWORD}" | kinit ${KRB_USERNAME}@CERN.CH
     - klist
-    - ssh ${KRB_USERNAME}@lxplus "mkdir -p ${BUILD_OUTPUT_ROOT_DIR}/${CI_PIPELINE_ID}_${CI_BUILD_REF}"
-    - scp -r cc7_rpms ${KRB_USERNAME}@lxplus:${BUILD_OUTPUT_ROOT_DIR}/${CI_PIPELINE_ID}_${CI_BUILD_REF}/centos7_x86_64
+    - ssh ${KRB_USERNAME}@lxplus "mkdir -p ${OUTPUT_PIPELINE_DIR}/repos"
+    - ssh ${KRB_USERNAME}@lxplus "ln -sfn ${OUTPUT_PIPELINE_DIR} `dirname ${OUTPUT_PIPELINE_DIR}`/latest" 
+    - rsync -av yumrepo/ ${KRB_USERNAME}@lxplus:${OUTPUT_PIPELINE_DIR}/repos/${OUTPUT_REPO_SUBDIR}
 
 
 build:slc6:
-  stage: build
+  <<: *build_job
   image: tswilliams/ipbus-sw-dev-slc6:latest
-  tags:
-    - docker
-  except:
-    - tags
-    - triggers
-  before_script:
-    - sudo yum -y install createrepo krb5-workstation openssh-clients
-    - mkdir -p ~/.ssh
-    - echo "HOST lxplus" > ~/.ssh/config
-    - echo "  StrictHostKeyChecking no" >> ~/.ssh/config
-    - echo "  GSSAPITrustDns yes" >> ~/.ssh/config
-    - echo "  GSSAPIAuthentication yes" >> ~/.ssh/config
-    - echo "  GSSAPIDelegateCredentials yes" >> ~/.ssh/config
-  script:
-    - env | grep -v PASSWORD | grep -v TOKEN
-    - make -k Set=all
-    - make -k Set=all rpm
-    - mkdir -p slc6_rpms
-    - cp `find . -iname "*.rpm"` slc6_rpms
-    - cp yumgroups-slc6.xml slc6_rpms/
-    - ls slc6_rpms
-    - createrepo -vg yumgroups-slc6.xml slc6_rpms
-    - echo "${KRB_PASSWORD}" | kinit ${KRB_USERNAME}@CERN.CH
-    - klist
-    - ssh ${KRB_USERNAME}@lxplus "mkdir -p ${BUILD_OUTPUT_ROOT_DIR}/${CI_PIPELINE_ID}_${CI_BUILD_REF}"
-    - scp -r slc6_rpms ${KRB_USERNAME}@lxplus:${BUILD_OUTPUT_ROOT_DIR}/${CI_PIPELINE_ID}_${CI_BUILD_REF}/slc6_x86_64
+  variables:
+    ADDITIONAL_RPMS_TO_INSTALL: ""
+    YUMGROUPS_FILE: "ci/yumgroups-slc6.xml"
+    OUTPUT_REPO_SUBDIR: "slc6_x86_64"
+
+
+build:centos7:
+  <<: *build_job
+  image: tswilliams/ipbus-sw-dev-cc7:latest
+  variables:
+    ADDITIONAL_RPMS_TO_INSTALL: "boost-devel"
+    YUMGROUPS_FILE: "ci/yumgroups-centos7.xml"
+    OUTPUT_REPO_SUBDIR: "centos7_x86_64"
+
 
 
 
 .job_template: &slc6_test_job
   stage: test
   image: tswilliams/ipbus-sw-dev-slc6:latest
   except:
-    - tags
     - triggers
   dependencies: []
   before_script:
-    - echo "[ipbus-sw-ci]" | sudo tee /etc/yum.repos.d/ipbus-sw.repo
-    - echo "name=CACTUS Project Software Repository" | sudo tee -a /etc/yum.repos.d/ipbus-sw.repo
-    - echo "baseurl=${BUILD_OUTPUT_ROOT_URL}/${CI_PIPELINE_ID}_${CI_BUILD_REF}/slc6_x86_64" | sudo tee -a /etc/yum.repos.d/ipbus-sw.repo
-    - echo "enabled=1" | sudo tee -a /etc/yum.repos.d/ipbus-sw.repo
-    - echo "gpgcheck=0" | sudo tee -a /etc/yum.repos.d/ipbus-sw.repo
+    - export OUTPUT_PIPELINE_URL=${OUTPUT_ROOT_URL}/commits/${CI_COMMIT_TAG:-${CI_COMMIT_SHA}}/pipeline${CI_PIPELINE_ID}
+    - if [ -n "${CI_COMMIT_TAG}" ]; then export OUTPUT_PIPELINE_URL=${OUTPUT_PIPELINE_URL/commits/tags} ; fi
+    - sudo cp ci/ipbus-sw-ci.repo /etc/yum.repos.d/ipbus-sw-ci.repo
+    - sudo sed -i -re "s|^baseurl=.*|baseurl=${OUTPUT_PIPELINE_URL}/repos/slc6_x86_64|g" /etc/yum.repos.d/ipbus-sw-ci.repo
     - sudo yum clean all
     - sudo yum -y groupinstall uhal
     - rpm -qa | grep cactus | sort
@@ -177,18 +156,15 @@ build:slc6:
   stage: test
   image: tswilliams/ipbus-sw-test-cc7:latest
   except:
-    - tags
     - triggers
   dependencies: []
   before_script:
-    - echo "[ipbus-sw-ci]" | sudo tee /etc/yum.repos.d/ipbus-sw.repo
-    - echo "name=CACTUS Project Software Repository" | sudo tee -a /etc/yum.repos.d/ipbus-sw.repo
-    - echo "baseurl=${BUILD_OUTPUT_ROOT_URL}/${CI_PIPELINE_ID}_${CI_BUILD_REF}/centos7_x86_64" | sudo tee -a /etc/yum.repos.d/ipbus-sw.repo
-    - echo "enabled=1" | sudo tee -a /etc/yum.repos.d/ipbus-sw.repo
-    - echo "gpgcheck=0" | sudo tee -a /etc/yum.repos.d/ipbus-sw.repo
+    - export OUTPUT_PIPELINE_URL=${OUTPUT_ROOT_URL}/commits/${CI_COMMIT_TAG:-${CI_COMMIT_SHA}}/pipeline${CI_PIPELINE_ID}
+    - if [ -n "${CI_COMMIT_TAG}" ]; then export OUTPUT_PIPELINE_URL=${OUTPUT_PIPELINE_URL/commits/tags} ; fi
+    - sudo cp ci/ipbus-sw-ci.repo /etc/yum.repos.d/ipbus-sw-ci.repo
+    - sudo sed -i -re "s|^baseurl=.*|baseurl=${OUTPUT_PIPELINE_URL}/repos/centos7_x86_64|g" /etc/yum.repos.d/ipbus-sw-ci.repo
     - sudo yum clean all
     - sudo yum -y groupinstall uhal
-    - sudo yum -y install boost-python
     - sudo yum -y install which
     - rpm -qa | grep cactus | sort
 

ci/ipbus-sw-ci.repo

@@ -0,0 +1,5 @@
+[ipbus-sw-ci]
+name=IPbus software repository for automated build
+baseurl=
+enabled=1
+gpgcheck=0

ci/ssh_config

@@ -0,0 +1,5 @@
+HOST lxplus
+  StrictHostKeyChecking no
+  GSSAPITrustDns yes
+  GSSAPIAuthentication yes
+  GSSAPIDelegateCredentials yes

yumgroups-centos7.xml → ci/yumgroups-centos7.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <comps>
   <group>
-    <id>cactus</id>
+    <id>ipbus-sw</id>
     <name>uhal</name>
     <default>true</default>
     <description>uHAL packages of the CACTUS YUM repository</description>

yumgroups-slc6.xml → ci/yumgroups-slc6.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <comps>
   <group>
-    <id>cactus</id>
+    <id>ipbus-sw</id>
     <name>uhal</name>
     <default>true</default>
     <description>uHAL packages of the CACTUS YUM repository</description>