Skip to content

Commit

Permalink
JBDS-3208 support running cleanup against an include filter so we nee…
Browse files Browse the repository at this point in the history 
…d only cleanup a single folder, not everyone else's folders too
  • Loading branch information
@nickboldt
nickboldt committed Mar 12, 2015
1 parent af04d45 commit cae60b2
Showing 1 changed file with 50 additions and 35 deletions.
85 changes: 50 additions & 35 deletions util/cleanup/jbosstools-cleanup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,20 @@ echo "" | tee -a $log
numbuildstokeep=1000 # keep X builds per branch
threshholdwhendelete=365 # purge builds more than X days old
dirsToScan="mars/snapshots/builds builds/staging/CI builds/nightly/core builds/nightly/coretests builds/nightly/soa-tooling builds/nightly/soatests builds/nightly/webtools builds/nightly/hibernatetools builds/nightly/integrationtests"
excludes="sftp>|empty_composite_site|*.*ml|\.properties|\.jar|\.zip|\.MD5|\.md5|\.blobstore|web|plugins|features|binary" # when dir matching, exclude *.*ml, *.properties, *.jar, *.zip, *.MD5, *.md5, web/features/plugins/binary/.blobstore
includes=""; # regex pattern to match within subdirs to make cleanup faster + more restrictive; eg., jbosstools-build-sites.aggregate.earlyaccess-site_master
delete=1 # if 1, files will be deleted. if 0, files will be listed for delete but not actually removed
checkTimeStamps=1 # if 1, check for timestamped folders, eg., 2012-09-30_04-01-36-H5622 and deduce the age from name. if 0, skip name-to-age parsing and delete nothing
childFolderSuffix="/" # for component update sites, set to "/"; for aggregate builds (not update sites) use "/all/repo/"
regenMetadataOnly=0 # set to 1 if only regenerating metadata, not cleaning up old build folders
noSubDirs=0 # normally, we want to scan for subdirs, but for a project like Locus, there's one less level of nesting so we need to override this with noSubDirs=1

if [[ $# -lt 1 ]]; then
echo "Usage: $0 [-k num-builds-to-keep] [-a num-days-at-which-to-delete] [-d dirs-to-scan] [--regen-metadata-only] [--childFolderSuffix /all/repo/]"
echo "Example (Jenkins job): $0 --keep 1 --age-to-delete 2 --childFolderSuffix /all/repo/"
echo "Example (publish.sh): $0 -k 5 -a 5 -S /all/repo/"
echo "Example (promote.sh): $0 --dirs-to-scan 'updates/integration/indigo/soa-tooling/' --regen-metadata-only"
echo "Example (promote.sh): $0 --dirs-to-scan 'updates/integration//locus' --regen-metadata-only --no-subdirs"
echo "Usage: $0 [-k num-builds-to-keep] [-a num-days-at-which-to-delete] [-d dirs-to-scan] [-i subdir-include-pattern] [--regen-metadata-only] [--childFolderSuffix /all/repo/]"
echo "Example (Jenkins): $0 --keep 1 --age-to-delete 2 --childFolderSuffix /all/repo/"
echo "Example (publish.sh): $0 -k 5 -a 5 -S /all/repo/"
echo "Example (promote.sh): $0 --dirs-to-scan 'updates/integration/indigo/soa-tooling/' --regen-metadata-only"
echo "Example (promote.sh): $0 --dirs-to-scan 'updates/integration//locus' --regen-metadata-only --no-subdirs"
echo "Example (rsync.sh): $0 -k 2 -a 2 -S /all/repo/ -d mars/snapshots/builds --include jbosstools-build-sites.aggregate"
exit 1;
fi

Expand All @@ -45,6 +47,7 @@ while [[ "$#" -gt 0 ]]; do
'-k'|'--keep') numbuildstokeep="$2"; shift 1;;
'-a'|'--age-to-delete') threshholdwhendelete="$2"; shift 1;;
'-d'|'--dirs-to-scan') dirsToScan="$2"; shift 1;;
'-i'|'--include') includes="$2"; shift 1;;
'-S'|'--childFolderSuffix') childFolderSuffix="$2"; shift 1;;
'-M'|'--regen-metadata-only') delete=0; checkTimeStamps=0; regenMetadataOnly=1; shift 0;;
'-N'|'--no-subdirs') noSubDirs=1; shift 0;;
Expand All @@ -54,41 +57,53 @@ done

getSubDirs ()
{
getSubDirsReturn="";
tab="";
if [[ $1 ]]; then dir="$1"; else dir="/downloads_htdocs/tools/builds/nightly/"; fi
if [[ $2 ]] && [[ $2 -gt 0 ]]; then
lev=$2
while [[ $lev -gt 0 ]]; do
tab=$tab"> ";
(( lev-- ));
getSubDirsReturn=""
tab=""
includePattern=""
if [[ $1 ]]; then dir="$1"; else echo "No directory passed to getSubDirs()!"; fi
if [[ $dir ]]; then
if [[ $2 ]] && [[ $2 -gt 0 ]]; then
lev=$2
while [[ $lev -gt 0 ]]; do
tab=$tab"> ";
(( lev-- ));
done
fi
if [[ $3 ]]; then
includePattern="$3"
echo "${tab}Check $dir for dirs matching /${includePattern}/ ..." | tee -a $log
else
echo "${tab}Check $dir..." | tee -a $log
fi
tmp=`mktemp`
echo "ls $dir" > $tmp
dirs=$(sftp -b $tmp tools@filemgmt.jboss.org 2>/dev/null)
i=0
for c in $dirs; do #exclude *.*ml, *.properties, *.jar, *.zip, *.MD5, *.md5, web/features/plugins/binary/.blobstore
# old way... if [[ $i -gt 2 ]] && [[ $c != "sftp>" ]] && [[ ${c##*.} != "" ]] && [[ ${c##*/*.*ml} != "" ]] && [[ ${c##*/*.properties} != "" ]] && [[ ${c##*/*.jar} != "" ]] && [[ ${c##*/*.zip} != "" ]] && [[ ${c##*/*.MD5} != "" ]] && [[ ${c##*/*.md5} != "" ]] && [[ ${c##*/web} != "" ]] && [[ ${c##*/plugins} != "" ]] && [[ ${c##*/features} != "" ]] && [[ ${c##*/binary} != "" ]] && [[ ${c##*/.blobstore} != "" ]]; then
if [[ $i -gt 2 ]] && [[ ${c##*.} != "" ]] && [[ ! $(echo "$c" | egrep "${excludes}") ]]; then
# if no include pattern set, or pattern matches, include this folder.
if [[ ! $includePattern ]] || [[ $(echo "$c" | egrep "$includePattern") ]]; then
getSubDirsReturn=$getSubDirsReturn" "$c
fi
fi
(( i++ ))
done
rm -f $tmp
fi
echo "${tab}Check $dir..." | tee -a $log
tmp=`mktemp`
echo "ls $dir" > $tmp
dirs=$(sftp -b $tmp tools@filemgmt.jboss.org 2>/dev/null)
i=0
for c in $dirs; do #exclude *.xml, *.html, *.properties, *.jar, *.zip, *.MD5, *.md5, web/features/plugins/binary/.blobstore
if [[ $i -gt 2 ]] && [[ $c != "sftp>" ]] && [[ ${c##*.} != "" ]] && [[ ${c##*/*.*ml} != "" ]] && [[ ${c##*/*.properties} != "" ]] && [[ ${c##*/*.jar} != "" ]] && [[ ${c##*/*.zip} != "" ]] && [[ ${c##*/*.MD5} != "" ]] && [[ ${c##*/*.md5} != "" ]] && [[ ${c##*/web} != "" ]] && [[ ${c##*/plugins} != "" ]] && [[ ${c##*/features} != "" ]] && [[ ${c##*/binary} != "" ]] && [[ ${c##*/.blobstore} != "" ]]; then
getSubDirsReturn=$getSubDirsReturn" "$c
fi
(( i++ ))
done
rm -f $tmp
}

# Check for $type builds more than $threshhold days old; keep minimum $numkeep builds per branch
# Check for $somepath builds more than $threshhold days old; keep minimum $numkeep builds per branch
clean ()
{
type=$1 # builds/nightly or updates/development/juno/soa-tooling, etc.
somepath=$1 # builds/nightly or updates/development/juno/soa-tooling, etc.
numkeep=$2 # number of builds to keep per branch
threshhold=$3 # purge builds more than $threshhold days old
type=${type//\/\//\/}; # remove duplicate slashes in paths - replace all // with /
type=${type//\/\//\/}; # repeat to replace /// with /
echo "Check for $type builds more than $threshhold days old; keep minimum $numkeep builds per branch" | tee -a $log
somepath=${somepath//\/\//\/}; # remove duplicate slashes in paths - replace all // with /
somepath=${somepath//\/\//\/}; # repeat to replace /// with /
echo "Check for $somepath builds more than $threshhold days old; keep minimum $numkeep builds per branch" | tee -a $log

getSubDirs /downloads_htdocs/tools/$type/ 0
getSubDirs /downloads_htdocs/tools/$somepath/ 0 $includes
subdirs=$getSubDirsReturn

# special case for Locus builds - only work in subfolders (3 levels: /updates/integration/locus/x.y.z/, not sub-subfolders (5 levels: /updates/integration/kepler/core/project/x.y.z/)
Expand All @@ -101,7 +116,7 @@ clean ()
# echo "[${subdirCount}] Found $buildid"
echo $buildid >> $tmp
done
regenProcess ${subdirCount} /downloads_htdocs/tools/$type/
regenProcess ${subdirCount} /downloads_htdocs/tools/$somepath/
else # for everyone else, work in sub-subfolders
for sd in $subdirs; do
getSubDirs $sd 1
Expand Down Expand Up @@ -213,8 +228,8 @@ regenCompositeMetadata ()
}

# now that we have all the methods and vars defined, let's do some cleaning!
for dir in $dirsToScan; do
clean $dir $numbuildstokeep $threshholdwhendelete
for path in $dirsToScan; do
clean $path $numbuildstokeep $threshholdwhendelete
done

# purge temp folder
Expand Down

0 comments on commit cae60b2

Please sign in to comment.