From 565713ea78659babd93414fcfe6b068572c805aa Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Tue, 9 Feb 2021 17:22:20 -0600 Subject: [PATCH 01/10] Update unified mapfile link --- config/condor_mapfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/condor_mapfile b/config/condor_mapfile index cede8bc92..0777ecc8b 100644 --- a/config/condor_mapfile +++ b/config/condor_mapfile @@ -1,5 +1,5 @@ # HTCondor-CE Certificate Mapfile -# http://research.cs.wisc.edu/htcondor/manual/v8.6/3_8Security.html#SECTION00484000000000000000 +# https://htcondor.readthedocs.io/en/stable/admin-manual/security.html#the-unified-map-file-for-authentication # Using GSI authentication for certificates requires the issuer CAs to be # installed in /etc/grid-security/certificates. If you would also like to # authenticate VOMS attributes, *.lsc files should be installed in From c144d8738372b69c4e2beaa77a6d5ac369d827fe Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Tue, 9 Feb 2021 17:39:30 -0600 Subject: [PATCH 02/10] Uppdate unified mapfile link to point at a specific 8.9.11 version --- config/condor_mapfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/condor_mapfile b/config/condor_mapfile index 0777ecc8b..d956e73e4 100644 --- a/config/condor_mapfile +++ b/config/condor_mapfile @@ -1,5 +1,5 @@ # HTCondor-CE Certificate Mapfile -# https://htcondor.readthedocs.io/en/stable/admin-manual/security.html#the-unified-map-file-for-authentication +# https://htcondor.readthedocs.io/en/v8_9_11/admin-manual/security.html#the-unified-map-file-for-authentication # Using GSI authentication for certificates requires the issuer CAs to be # installed in /etc/grid-security/certificates. If you would also like to # authenticate VOMS attributes, *.lsc files should be installed in From a0702ba9ef0a03e2382df21d099b1d609235a588 Mon Sep 17 00:00:00 2001 From: Thomas Hartmann Date: Wed, 10 Feb 2021 14:27:00 +0100 Subject: [PATCH 03/10] python2 print remnant clashing with future print/python3 --- src/condor_ce_info_status | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/condor_ce_info_status b/src/condor_ce_info_status index e94bda57a..d2f11c6f8 100755 --- a/src/condor_ce_info_status +++ b/src/condor_ce_info_status @@ -171,7 +171,7 @@ def main(argv): if opts.verbose: for res in resources: - print res + print( res ) elif opts.submitfile: resources = [x for x in resources if x.get('Name')] if not resources: From bfd27ffbfdc678aa26e0417627038ab8e1214fe9 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Wed, 10 Feb 2021 13:51:36 -0600 Subject: [PATCH 04/10] Style change --- src/condor_ce_info_status | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/condor_ce_info_status b/src/condor_ce_info_status index d2f11c6f8..221b087e4 100755 --- a/src/condor_ce_info_status +++ b/src/condor_ce_info_status @@ -171,7 +171,7 @@ def main(argv): if opts.verbose: for res in resources: - print( res ) + print(res) elif opts.submitfile: resources = [x for x in resources if x.get('Name')] if not resources: From adf9511fca75960449f2a351f054c4a1bd845f90 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Thu, 21 Jan 2021 14:42:02 -0600 Subject: [PATCH 05/10] Initial upload script updates for GHA --- tests/build_rpms.sh | 7 +++---- tests/sftp_upload.sh | 34 ++++++++++------------------------ 2 files changed, 13 insertions(+), 28 deletions(-) diff --git a/tests/build_rpms.sh b/tests/build_rpms.sh index 47f442288..d7e39056d 100755 --- a/tests/build_rpms.sh +++ b/tests/build_rpms.sh @@ -61,7 +61,6 @@ rpmbuild --define '_topdir /tmp/rpmbuild' --define 'dist %{nil}' -bs /tmp/rpmbui rpmbuild --define '_topdir /tmp/rpmbuild' -bb /tmp/rpmbuild/SPECS/htcondor-ce.spec # dir needs to be inside htcondor-ce so it's visible outside the container -mkdir -p htcondor-ce/travis_deploy -cp -f /tmp/rpmbuild/RPMS/*/*.rpm htcondor-ce/travis_deploy/ -cp -f /tmp/rpmbuild/SRPMS/*.rpm htcondor-ce/travis_deploy/ - +mkdir -p htcondor-ce/ci_deploy +cp -f /tmp/rpmbuild/RPMS/*/*.rpm htcondor-ce/ci_deploy/ +cp -f /tmp/rpmbuild/SRPMS/*.rpm htcondor-ce/ci_deploy/ diff --git a/tests/sftp_upload.sh b/tests/sftp_upload.sh index a41435b07..1b783aabc 100755 --- a/tests/sftp_upload.sh +++ b/tests/sftp_upload.sh @@ -14,13 +14,8 @@ if [[ -r $progdir/env.sh ]]; then set $_old_x fi -if [[ $TRAVIS_PULL_REQUEST != false ]]; then - echo "Not running deploy on a PR" - exit 0 -fi - -project=${TRAVIS_REPO_SLUG#*/} -repo_owner=${TRAVIS_REPO_SLUG%/*} +project=${GITHUB_REPOSITORY#*/} +repo_owner=${GITHUB_REPOSITORY%/*} keyfile=$progdir/id_rsa_cibot2 upload_server=ci-xfer.chtc.wisc.edu @@ -30,19 +25,9 @@ hostsig="ci-xfer.chtc.wisc.edu ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDyrceRMLPsO function setup_ssh_to_chtc { - if [[ ! -e $keyfile.enc.$repo_owner ]]; then - echo "Repo owner $repo_owner does not have a key in the repo." >&2 - echo "Cannot deploy via ssh." >&2 - return 1 - fi ( umask 077 mkdir -p ~/.ssh - openssl aes-256-cbc \ - -K $encrypted_e14a22ad945b_key \ - -iv $encrypted_e14a22ad945b_iv \ - -in $keyfile.enc.$repo_owner -out $keyfile \ - -d cat > ~/.ssh/config <<__END__ Host $upload_server User cibot @@ -54,25 +39,26 @@ ChallengeResponseAuthentication no KerberosAuthentication no IdentitiesOnly yes __END__ + printf "%s\n" "${CI_XFER_KEY}" > ~/.ssh/id_rsa_cibot2 printf "%s\n" "$hostsig" > ~/.ssh/known_hosts ) } function sftp_to_chtc { local ret=0 - local remote_dir=/var/tmp/travis/$repo_owner/$project - if [[ -n ${TRAVIS_TAG-} ]]; then + local remote_dir=/var/tmp/ci_deploy/$repo_owner/$project + if [[ $GITHUB_REF =~ ^refs/tags/ ]]; then # .../htcondor-ce-v2.3.4 - remote_dir=${remote_dir}-$(tr / _ <<<"$TRAVIS_TAG") + remote_dir=${remote_dir}-$(tr / _ <<<"${GITHUB_REF##refs/tags/}") else # .../htcondor-ce-88 - remote_dir=${remote_dir}-${TRAVIS_BUILD_NUMBER} + remote_dir=${remote_dir}-${GITHUB_RUN_ID} fi set +x script=$(mktemp -t build_rpms.$$.XXXXXX) cat >>"$script" <<__END__ --MKDIR /var/tmp/travis --MKDIR /var/tmp/travis/$repo_owner +-MKDIR /var/tmp/ci_deploy +-MKDIR /var/tmp/ci_deploy/$repo_owner -MKDIR $remote_dir CD $remote_dir __END__ @@ -91,7 +77,7 @@ __END__ setup_ssh_to_chtc -sftp_to_chtc "$progdir"/../travis_deploy/* +sftp_to_chtc "$progdir"/../ci_deploy/* # vim:et:sw=4:sts=4:ts=8 From 3289e0850292fe92aae23f868eba0ed237e767db Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Thu, 21 Jan 2021 15:09:52 -0600 Subject: [PATCH 06/10] Initial GHA uploader commit --- .github/workflows/upload_rpms.yaml | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/upload_rpms.yaml diff --git a/.github/workflows/upload_rpms.yaml b/.github/workflows/upload_rpms.yaml new file mode 100644 index 000000000..64da24034 --- /dev/null +++ b/.github/workflows/upload_rpms.yaml @@ -0,0 +1,38 @@ +name: Build and upload HTCondor-CE RPMs +on: + release: + # there seems to be contention between 'published' vs 'created' here + # 'published' occurs with a GitHub release + types: [published] + +jobs: + build-rpms-and-upload: + runs-on: ubuntu-latest + if: startsWith(github.repository, 'htcondor/') + strategy: + matrix: + dver: [7, 8] + steps: + - uses: actions/checkout@v2 + + - name: Prepare Docker + run: | + echo 'DOCKER_OPTS="-H tcp://127.0.0.1:2375 -H unix:///var/run/docker.sock -s devicemapper"' | sudo tee /etc/default/docker > /dev/null && + sudo service docker restart + + - name: Start CentOS ${{ matrix.dver}} image + run: | + docker run --detach --env "container=docker" \ + --name $GITHUB_SHA \ + --volume `pwd`:/htcondor-ce:rw \ + centos:centos${{ matrix.dver }} \ + /usr/sbin/init + + - name: Build CHTC EL${{ matrix.dver }} RPMs + run: | + docker exec $GITHUB_SHA \ + /bin/bash -xc \ + "/htcondor-ce/tests/build_rpms.sh \ + ${{ matrix.dver }} \ + uw_build" + From 5f5567850798e52bac8582a6eebf4fca61ea1f83 Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Thu, 21 Jan 2021 15:52:22 -0600 Subject: [PATCH 07/10] Move SSH key handling to ssh-agent outside of the script --- .github/workflows/upload_rpms.yaml | 7 +++++++ tests/sftp_upload.sh | 3 --- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/upload_rpms.yaml b/.github/workflows/upload_rpms.yaml index 64da24034..e50fd41d3 100644 --- a/.github/workflows/upload_rpms.yaml +++ b/.github/workflows/upload_rpms.yaml @@ -36,3 +36,10 @@ jobs: ${{ matrix.dver }} \ uw_build" + - name: Upload CHTC EL${{ matrix.dver }} RPMs + run: | + ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null + ssh-add - <<< "${{ secrets.CI_XFER_KEY }}" + tests/sftp_upload.sh + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock diff --git a/tests/sftp_upload.sh b/tests/sftp_upload.sh index 1b783aabc..5ad137087 100755 --- a/tests/sftp_upload.sh +++ b/tests/sftp_upload.sh @@ -17,7 +17,6 @@ fi project=${GITHUB_REPOSITORY#*/} repo_owner=${GITHUB_REPOSITORY%/*} -keyfile=$progdir/id_rsa_cibot2 upload_server=ci-xfer.chtc.wisc.edu # from "ssh-keyscan -t rsa ci-xfer.chtc.wisc.edu" hostsig="ci-xfer.chtc.wisc.edu ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDyrceRMLPsOmdtDHxXpfI82snDF0Q9/Z1Mick5zsQK1RyOtNgkyvXM50AJSPPSl0I9JmIxSBxhqcNDcbDz0Kc8tKcA1iGQxp4Ll9z9ZCl60AUq72WwkS1A4z11JjRoYvw1CL8bvoJhk55dcgAz+bXWx/eTwcBsmW80/okNDkdYmtv+QgfUmRP6TjMtIkzvCsXi5x+B4j66yQcLDDYb36EcGyHZqoyLuxkxX0OwS7LuzDfnKxpsV9jlnu3PuJnZOizalqKUpTYc2b83XnfsIYTqoiclmFr89+WuQJG6e/596y/9aVtNacCphdS7u3D+tSoME6OG7xQtZiQfkWvKPicv" @@ -31,7 +30,6 @@ function setup_ssh_to_chtc { cat > ~/.ssh/config <<__END__ Host $upload_server User cibot -IdentityFile $keyfile PubkeyAuthentication yes PasswordAuthentication no GSSAPIAuthentication no @@ -39,7 +37,6 @@ ChallengeResponseAuthentication no KerberosAuthentication no IdentitiesOnly yes __END__ - printf "%s\n" "${CI_XFER_KEY}" > ~/.ssh/id_rsa_cibot2 printf "%s\n" "$hostsig" > ~/.ssh/known_hosts ) } From 58d55bf608566781a2fc5e727e0e5a0904c4da0b Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Mon, 25 Jan 2021 16:58:33 -0600 Subject: [PATCH 08/10] Allow use of SSH agent for keys --- tests/sftp_upload.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/sftp_upload.sh b/tests/sftp_upload.sh index 5ad137087..0d0508dc1 100755 --- a/tests/sftp_upload.sh +++ b/tests/sftp_upload.sh @@ -35,7 +35,6 @@ PasswordAuthentication no GSSAPIAuthentication no ChallengeResponseAuthentication no KerberosAuthentication no -IdentitiesOnly yes __END__ printf "%s\n" "$hostsig" > ~/.ssh/known_hosts ) From 9e72be11ca3d3ac899899d743ea52b6916ca1fea Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Mon, 25 Jan 2021 16:58:47 -0600 Subject: [PATCH 09/10] Remove unrecognized auth method in GHA "/home/runner/.ssh/config line 7: Unsupported option "kerberosauthentication" --- tests/sftp_upload.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/sftp_upload.sh b/tests/sftp_upload.sh index 0d0508dc1..8f85c70b5 100755 --- a/tests/sftp_upload.sh +++ b/tests/sftp_upload.sh @@ -34,7 +34,6 @@ PubkeyAuthentication yes PasswordAuthentication no GSSAPIAuthentication no ChallengeResponseAuthentication no -KerberosAuthentication no __END__ printf "%s\n" "$hostsig" > ~/.ssh/known_hosts ) From 2cb97e11d9fd2a06e093a23e9adc23a98d35829a Mon Sep 17 00:00:00 2001 From: Brian Lin Date: Mon, 8 Feb 2021 16:01:39 -0600 Subject: [PATCH 10/10] Allow interactive SSH method to force the PAM chain --- tests/sftp_upload.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sftp_upload.sh b/tests/sftp_upload.sh index 8f85c70b5..be86df033 100755 --- a/tests/sftp_upload.sh +++ b/tests/sftp_upload.sh @@ -31,7 +31,7 @@ function setup_ssh_to_chtc { Host $upload_server User cibot PubkeyAuthentication yes -PasswordAuthentication no +PasswordAuthentication yes GSSAPIAuthentication no ChallengeResponseAuthentication no __END__