/
atom_data_download.yml
85 lines (71 loc) · 3.04 KB
/
atom_data_download.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
trigger:
- none
pr:
- none
variables:
system.debug: 'true'
ATOMIC_DATA_URL: "http://kookaburra.phyast.pitt.edu/hillier/cmfgen_files/atomic_data_15nov16.tar.gz"
SSH_REPO: git@github.com:tardis-sn/carsus.git
CMFGEN_DIR: /usr/share/ATOMIC_DATA/atomic
PYTHON_VERSION: 3.7
jobs:
- job: "Fetch_Atomic_Data"
# Agent pool and vm image
pool:
vmImage: "Ubuntu-16.04"
steps:
# Secure files stored in the azure server are encryped and again decrypted by the azure task that uses the file
# Download a secure file to a temporary location in the virtual machine
- task: DownloadSecureFile@1
inputs:
secureFile: 'id_carsus_rsa'
# Make sure you've added the generated private key file (named 'id_azure_rsa' here) to library & authorize it for all pipelines
# by using: https://docs.microsoft.com/en-us/azure/devops/pipelines/library/secure-files?view=azure-devops#how-do-i-authorize-a-secure-file-for-use-in-all-pipelines
# Install an SSH key prior to a build or release
# This is needed to give azure access to deploy to github
# hostName is the line that was added to ~/.ssh/known_hosts when you added the RSA host key. (Output of ssh-keyscan should look something like: [1]As3..=ssh-rsa ..)
# sshPublicKey should be a string value of what is inside your .pub file (i.e: rsa-key Axddd... username@server)
# sshKeySecureFile is the downloaded secure file you generated
- task: InstallSSHKey@0
inputs:
knownHostsEntry: $(gh_host)
sshPublicKey: $(public_key)
#sshPassphrase: # Optional - leave empty if it was left empty while generating the key
sshKeySecureFile: 'id_carsus_rsa'
- bash: |
git config --local user.name "Azure Pipelines"
git config --local user.email "azuredevops@microsoft.com"
git checkout atom_data_pipeline || git checkout --orphan atom_data_pipeline
displayName: "Configure github"
# Download atomic data for tests
- bash: |
mkdir /usr/share/ATOMIC_DATA
wget $ATOMIC_DATA_URL -O /usr/share/ATOMIC_DATA/ATOMIC_DATA.gz
displayName: "Fetch atomic data"
- bash: |
hash_check_1=`cat hash_check_1`
echo $hash_check_1
hash_check_2=`md5sum /usr/share/ATOMIC_DATA/ATOMIC_DATA.gz`
echo $hash_check_2
if [ "$hash_check_1" = "$hash_check_2" ]; then
echo 'No need to update data'; else
sed -i "s+$hash_check_1+$hash_check_2+g" hash_check_1
tar -zxvf /usr/share/ATOMIC_DATA/ATOMIC_DATA.gz -C /usr/share/ATOMIC_DATA
fi
displayName: "Extract data and update hash check if updated"
# Python script
# Run a Python file or inline script
- task: PythonScript@0
inputs:
scriptSource: filePath
scriptPath: 'carsus/io/cmfgen/hdfgen.py'
# Push to github
- bash: |
git add --all
if git diff --staged --quiet; then
echo "exiting with no docs changes"
exit 0
fi
git commit -m "Add new reference data and update hash code"
git push $(SSH_REPO) atom_data_pipeline
displayName: "Push to github branch"