Skip to content

Commit

Permalink
Merge "Fix problem with publishing .tar.gz files to the CDN"
Browse files Browse the repository at this point in the history
  • Loading branch information
Trafodion Jenkins authored and Gerrit Code Review committed Mar 4, 2015
2 parents 699deba + 3d79793 commit 354f65a
Show file tree
Hide file tree
Showing 5 changed files with 123 additions and 78 deletions.
1 change: 1 addition & 0 deletions modules/traf/files/jenkins/jenkins-sudo-cdn.sudo
View file
@@ -0,0 +1 @@
jenkins ALL = NOPASSWD:/usr/local/bin/useObjectStorage.sh
2 changes: 1 addition & 1 deletion modules/traf/files/jenkins_job_builder/config/macros.yaml
View file
Expand Up @@ -493,7 +493,7 @@
do
# make sure there are actually files
if [[ $(find $dirname -type f | wc -l) -ne 0 ]]; then
/usr/local/bin/useObjectStorage.sh -cu $dirName
sudo -n /usr/local/bin/useObjectStorage.sh -cu $dirName
(( res += $? ))
fi
done
Expand Down
2 changes: 1 addition & 1 deletion modules/traf/manifests/base.pp
View file
Expand Up @@ -125,7 +125,7 @@
ensure => file,
owner => 'root',
group => 'root',
mode => '0755',
mode => '0750',
content => template('traf/useObjectStorage.sh.erb'),
}

Expand Down
9 changes: 9 additions & 0 deletions modules/traf/manifests/slave.pp
View file
Expand Up @@ -113,6 +113,15 @@
require => [ Sshkey['logs.trafodion.org'], Mount['/home/jenkins'] ],
}

# allow Jenkins to run script to upload files to CDN
file { '/etc/sudoers.d/jenkins-sudo-cdn':
ensure => present,
source => 'puppet:///modules/traf/jenkins/jenkins-sudo-cdn.sudo',
owner => 'root',
group => 'root',
mode => '0440',
}


include jenkins::cgroups

Expand Down
187 changes: 111 additions & 76 deletions modules/traf/templates/useObjectStorage.sh.erb
View file
Expand Up @@ -23,19 +23,19 @@
export PRGNAME=$(basename $0)
scriptDir=$(dirname $0)
if [[ $scriptDir == "." ]]; then scriptDir=`pwd`; fi
backupContainer="BackupContainer"
cdnContainer="trafcdn"
fName=""
outputFile=""
container=""
export backupContainer="BackupContainer"
export cdnContainer="trafcdn"
export container=""
containerSub=""
tfName=""
outputFile=""
swiftVerb=""
swiftFile=""
swiftCmdArgs=""
res=0

# set up environment to use swift
swift=/usr/local/bin/swift
export swift=/usr/local/bin/swift
export OS_REGION_NAME=region-a.geo-1
export OS_DNS_SERVICE_TYPE=hpext:dns
export OS_AUTH_URL=https://region-a.geo-1.identity.hpcloudsvc.com:35357/v2.0/
Expand Down Expand Up @@ -82,6 +82,94 @@ ProgramHelp() {
"
}

#-----------------------------------------
# Desc : Upload a file to object storage
# Usage : SwiftUpload <file_name>
#-----------------------------------------

SwiftUpload() {
fName=$1
uploadOptions="--retries=3 --changed --skip-identical"
headerOptions=""
host=$(hostname -s)

# check if file size is greater than 4.99 GB (5,358,000,000 bytes)
# if file size is too big then segment the file to 2 GB chunks
fSize=$(stat -c%s "$fName")
if [[ $fSize -gt 5358000000 ]]; then
uploadOptions+=" -S 2147483648"
fi

# set options based on container
if [[ $container = "$backupContainer" ]]; then
# check to see if fName contains absolute path
if [[ -n ${fName%%/*} ]]; then
echo "ERROR: Must use absolute path when uploading to the container $backupContainer"
exit 1
else
# get current md5sum of file
fMd5sum=$(md5sum $fName | awk '{print $1}')

# get md5sum of file from the cloud
fCloudMd5sum=$($swift stat $container ${host}${fName} 2>/dev/null | grep "ETag:" | awk '{print $2}')

if [[ "$fMd5sum" = "$fCloudMd5sum" ]]; then
echo "INFO: File $fName has not changed since the last upload. Skipping upload ..."
rc=$?
else
# if fCloudMd5sum is not empty then file has been uploaded before
if [[ -n $fCloudMd5sum ]]; then
# set header X-Delete-After to 30 days so we do not keep
# more than 30 days worth of previous version of the file
echo "INFO: Setting $fName to expire in 30 days ..."
$swift post -H 'x-delete-after: 259200' $container ${host}${fName}
rc=$?
fi

echo "INFO: Uploading file $fName to container $container ..."
$swift upload $uploadOptions $headerOptions $container/$host $fName
(( rc += $? ))
fi
fi
elif [[ $container = "$cdnContainer" ]]; then
# check to see if fName starts with daily, old-release, opencart, pre-release, release, or sandbox
if [[ $fName =~ ^daily/.*|^old-release/.*|^opencart/.*|^pre-release/.*|^release/.*|^sandbox/.* ]]; then
# Swift sets content-type for *.tar.gz files to application/x-tar
# This is the correct content-type. However different browsers handle
# this file type differently which causes the file name to get mangled
# by the browser. We will set the content-type to application/octet-stream
# to force the browsers to always download these files with the correct extension
if [[ $fName =~ .*.tar.gz ]]; then
headerOptions+=" -H content-type:application/octet-stream"
fi

# if daily or pre-release build
# set header X-Delete-After to 30 days so we do not keep
# more than 30 days worth of daily builds
if [[ $fName =~ ^daily/.*|^pre-release/.* ]]; then
echo "INFO: Setting $fName to expire in 30 days ..."
headerOptions+=" -H x-delete-after:259200"
fi

echo "INFO: Uploading $fName to container $container ..."
$swift upload $uploadOptions $headerOptions $container $fName
rc=$?

# There appears to be a bug in the swift client. The header
# content-type is ignored when passed via upload.
# run a POST command to make sure it is set
echo "INFO: Setting file content-type again to be sure it is set ..."
$swift post $headerOptions $container $fName
(( rc += $? ))
else
echo "ERROR: Path must start with one of the following (daily/, old-release/, opencart/, pre-release/, release/, or sandbox/)"
echo " when uploading to the container $cdnContainer"
exit 1
fi
fi
return $rc
}


#======================
# Main Program
Expand All @@ -107,7 +195,7 @@ else
;;
u) # Upload file
swiftVerb="upload"
fName="$OPTARG"
tfName="$OPTARG"
;;
d) # Download file
swiftVerb="download"
Expand Down Expand Up @@ -152,77 +240,24 @@ elif [[ $swiftVerb = "download" ]]; then
fi
res=$?
elif [[ $swiftVerb = "upload" ]]; then
uploadOptions="--retries=3 --changed --skip-identical"
host=$(hostname -s)

# Swift sets content-type for *.tar.gz files to application/x-tar
# This is the correct content-type. However different browsers handle
# this file type differently which causes the file name to get mangled
# by the browser. We will set the content-type to application/octet-stream
# to force the browsers to always download these files with the correct extension
if [[ $fName =~ .*.tar.gz ]]; then
uploadOptions+=" -H content-type:application/octet-stream"
fi

if [[ -f $fName ]]; then
# check if file size is greater than 4.99 GB (5,358,000,000 bytes)
# if file size is too big then segment the file to 2 GB chunks
fSize=$(stat -c%s "$fName")
if [[ $fSize -gt 5358000000 ]]; then
uploadOptions+=" -S 2147483648"
fi

# get current md5sum of file
fMd5sum=$(md5sum $fName | awk '{print $1}')
fi

# set options based on container
if [[ $container = "$backupContainer" ]]; then
# check to see if fName contains absolute path
if [[ -n ${fName%%/*} ]]; then
echo "ERROR: Must use absolute path when uploading to the container $backupContainer"
exit 1
else
# get md5sum of file from the cloud
fCloudMd5sum=$($swift stat $container ${host}${fName} 2>/dev/null | grep "ETag:" | awk '{print $2}')

if [[ "$fMd5sum" = "$fCloudMd5sum" ]]; then
echo "INFO: File $fName has not changed since the last upload. Skipping upload ..."
res=$?
else
# if fCloudMd5sum is not empty then file has been uploaded before
if [[ -n $fCloudMd5sum ]]; then
# set header X-Delete-After to 30 days so we do not keep
# more than 30 days worth of previous version of the file
echo "INFO: Setting $fName to expire in 30 days ..."
$swift post -H 'x-delete-after: 259200' $container ${host}${fName}
res=$?
fi

echo "INFO: Uploading file $fName to container $container ..."
$swift upload $uploadOptions $container/$host $fName
(( res += $? ))
fi
fi
elif [[ $container = "$cdnContainer" ]]; then
# Normally swift does not care whether you are uploading a file
# or a directory. However, since we need special processing
# for *.tar.gz files we need to process each file in a directory
#
# Check if $tfName is a directory and process accordingly
if [[ ! -d $tfName ]]; then
echo "INFO: $tfName is a file ..."
SwiftUpload $tfName
res=$?
else
echo "INFO: $tfName is a directory ..."
# since fName can be a directory check to make sure that fName actually contains files to be uploaded
if [[ $(find $fName -type f | wc -l) -ne 0 ]]; then
# check to see if fName starts with daily, old-release, opencart, pre-release, release, or sandbox
if [[ $fName =~ ^daily/.*|^old-release/.*|^opencart/.*|^pre-release/.*|^release/.*|^sandbox/.* ]]; then
if [[ $fName =~ ^daily/.*|^pre-release/.* ]]; then
# set header X-Delete-After to 30 days so we do not keep
# more than 30 days worth of daily builds
echo "INFO: Setting $fName to expire in 30 days ..."
uploadOptions+=" -H x-delete-after:259200"
fi
echo "INFO: Uploading $fName to container $container ..."
$swift upload $uploadOptions $container $fName
if [[ $(find $tfName -type f | wc -l) -ne 0 ]]; then
find $tfName -type f | while read tFile
do
SwiftUpload $tFile
res=$?
else
echo "ERROR: Path must start with one of the following (daily/, old-release/, opencart/, pre-release/, release/, or sandbox/)"
echo " when uploading to the container $cdnContainer"
exit 1
fi
done
else
echo "WARNING: There were no files found in $fName. Upload to container $cdnContainer cancelled."
fi
Expand Down

0 comments on commit 354f65a

Please sign in to comment.