/
hpss_to_mpdrive.sh
executable file
·63 lines (53 loc) · 2.01 KB
/
hpss_to_mpdrive.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
#!/bin/bash
input=$2
[[ ! -e $input ]] && echo $input missing && exit
dirs=`awk -F/ '{print $1}' $input | sort -u`
cd $1 && pwd
stage_dir=rclone_to_mp_drive
[[ ! -d $stage_dir ]] && mkdir -pv $stage_dir
for dir in $dirs; do
echo $dir
files=`grep "^$dir" $input`
echo $files | tr ' ' '\n' | sort -u > ${dir}.files
wc -l ${dir}.files
rclone lsf -R --files-only mp-drive:calculations/garden/$dir | sed "s:^:$dir/:g" | sed 's:.tar.gz::g' | sort -u > ${dir}.rclone_lsf
wc -l ${dir}.rclone_lsf
missing_paths=${dir}.paths
[[ -e $missing_paths ]] && rm -v $missing_paths
for f in $(comm --check-order -23 ${dir}.files ${dir}.rclone_lsf); do # launch dirs missing in mp-drive
launch_dir_tar="${stage_dir}/${f}.tar.gz"
if [[ ! -f $launch_dir_tar || ! -s $launch_dir_tar ]]; then
echo $f >> $missing_paths
elif [ -d $f ]; then
rm -rv $f
fi
done
for f in $(comm --check-order -12 ${dir}.files ${dir}.rclone_lsf | tr '\n' ' '); do # already cloned launch dirs -> cleanup
launch_dir_tar="${stage_dir}/${f}.tar.gz"
[[ -d $f ]] && rm -rv $f
[[ -e $launch_dir_tar ]] && rm -v $launch_dir_tar
done
rm -v ${dir}.files ${dir}.rclone_lsf
[[ ! -e $missing_paths ]] && echo nothing missing on GDrive!? && continue
wc -l $missing_paths
htar -xvf garden/${dir}.tar `cat $missing_paths | tr '\n' ' '`
ls -ltrhd ${dir}
[[ $? -ne 0 ]] && echo missing paths not found in HPSS!? && continue
for f in `cat $missing_paths`; do
[[ ! -e $f ]] && echo $f not found in HPSS!? && continue
launch_dir_tar="${stage_dir}/${f}.tar.gz"
echo $launch_dir_tar ...
mkdir -p `dirname $launch_dir_tar`
if tar --use-compress-program="pigz -9rv" -cf $launch_dir_tar -C `dirname $f` `basename $f`; then
ls -ltrh $launch_dir_tar
else
echo 'problem with launch dir tar!'
rm -v $launch_dir_tar
exit
fi
[[ -d $f ]] && rm -rv $f
done
rm -v $missing_paths
rclone -v copy $stage_dir/$dir mp-drive:calculations/garden/$dir
find $dir -type d -empty -print -delete
done