ddm

#!/bin/bash

VERSION='0.4'

# DDM
# See included documentation

# Written by Dieter Plaetinck
# http://dieter.plaetinck.be
# http://github.com/Dieterbe/ddm
# This code is licensed under GPL v3. See http://www.gnu.org/licenses/gpl-3.0.txt

 
 #
 # SOME MISC FUNCTIONS
 #

usage()
{
cat << EOF
This is ddm $VERSION
usage: `basename $0` -a <action> [more options]
       `basename $0` -h

OPTIONS:
   -a <action>         Action         $ACTION_HELPSTRING
   -d <dataset-name>   Dataset name   $DATASET_NAME_HELPSTRING
   -t <dataset-type>   Dataset type   $DATASET_TYPE_HELPSTRING
   -r <repo-type>      Repo type      $REPOSITORY_TYPE_HELPSTRING
   -m <message>        Message        $MESSAGE_HELPSTRING
   -p                  Pretend
   -v                  Verbose
   -h                  Show help      (this message)
EOF
}


ask_user ()
{
	# $1 : question to ask
	# $2 : 1 for yes/no question, just check if  $user_response='y' afterwards
	
	user_response= # this variable will be read by the caller.

	[ -z "$1" ] && echo_die "ask_user called without question to ask" 100
	
	if [ "$2" == '1' ]
	then
		echo "$1 (y/n)"
	else
		echo "$1"
	fi
		
	read user_response
	user_response=`tr "[:upper:]" "[:lower:]" <<< $user_response`
	
	[ "$2" == '1' ] && [ "$user_response" == 'y' -o "$user_response" == 'yes' ] && user_response='y'
}


# echo's the string if at least verbose or pretend is enabled

echo_pretend_verbose () 
{
	if [ "$PRETEND" == '1' ]
	then
		echo -e "$@ (pretend)"
	else
		echo_verbose "$@"
	fi
}


echo_pretend ()
{
	if [ "$PRETEND" == '1' ]
	then
		echo -e "$@ (pretend)"
	fi
}


echo_verbose ()
{
	if [ -n "$VERBOSE" ]
	then
		#TODO: emits a newline too many when there is a newline at the end of "$@" such as is the case when echo_verbose_buffer is echo'd. to be fixed someday
		#echo -e "${@/\n\$/}"
		echo -e "$@"
	fi
}


echo_debug ()
{
	if [ "$DEBUG" == "1" ] #export this variable or set it through ddmrc.
	then
		echo "DEBUG: $@"
	fi
	
	return 0
}


echo_debug_vars ()
{
	echo_debug "	Variable dump:
		REPOSITORY_FULL $REPOSITORY_FULL [ REPOSITORY_TYPE $REPOSITORY_TYPE ]	
		DATASET_LOCAL_REL $DATASET_LOCAL_REL - DATASET_LOCAL_FULL $DATASET_LOCAL_FULL - DATASET_LOCAL_PARENT_FULL $DATASET_LOCAL_PARENT_FULL
		DATASET_LOCAL_BASE $DATASET_LOCAL_BASE - DATASET_LOCAL_NAME $DATASET_LOCAL_NAME [ DATASET_LOCAL_TYPE $DATASET_LOCAL_TYPE ]"      

}


echo_die ()
{
	echo "$1" >&2
	[ -n "$2" ] && exit $2
	echo "No exit code set when calling echo_die"
	exit 100
}


check_is_in()
{
	test=${1:-unknown}
	allowed=${2:-unknown}
	type=${3:-unknown}
	exitcode=${4:-100} #exit code to use when test fails
	
	found=0
	
	for all in $allowed
	do
		if [ $all == $test ]
		then
			found=1
		fi
	done
	
	if [ $found -eq 0 ]
	then
		echo_die "$test is not an allowed/known $type (allowed: $allowed)" $exitcode
	fi
}


check_writable_dir ()
{

	[ -z "$1" -o -z "$2" ] && echo_debug "No path or type given when calling check_writable_dir" && exit 100

	path="$1"
	type="$2"
	exitcode=${3:-100}
	
	if [ ! -d "$path" ]
	then
		echo_die "$2 $path does not exist or is not a directory" $exitcode
	elif [ ! -r "$path" ]
	then
		echo_die "$2 $path is an existing directory, but is not readable" $exitcode
	elif [ ! -x "$path" ] # not sure if this is really needed...
	then
		echo_die "$2 $path is a readable directory, but not executable" $exitcode
	fi
}


check_repo_path ()
{
	[ -z "$1" -o -z "$2" ] && echo_debug "no path or repotype given when calling check_repo_path" && exit 100
	path=$1
	repotype=$2
	exitcode=3
	
	if [ "$repotype" == 'svn' ]
	then
		if [ "$IGNORE_DATASET_REMOTE_SVN" == '0' ]
		then
			output=`wrap_svn info $path 2>&1`
			if [ $? -gt 0 ]
			then
				echo "Something appears to be wrong with the $repotype repo $path :"
				echo_die "$output" $exitcode
			fi
		fi
	elif [ "$repotype" == 'vfs' ]
	then
		check_writable_dir "$path" "$repotype repo" $exitcode
	fi
}


check_dataset_path ()
{
	[ -z "$1" -o -z "$2" ] && echo_debug "no path or repotype given when calling check_dataset_path" && exit 100
	path=$1
	repotype=$2
	exitcode=2
	
	if [ "$repotype" == 'svn' ]
	then
		output=`wrap_svn info $path 2>&1`
		if [ $? -gt 0 ]
		then
			echo "Something appears to be wrong with the $repotype dataset $path :"
			echo_die "$output" $exitcode
		fi

		if [ "$IGNORE_DATASET_REMOTE_SVN" == '0' ]
		then
			remote=`echo "$output" | grep 'URL: ' | awk '{print $2}'`
			if [ "$remote" != "$REPOSITORY_FULL" ]
			then
				echo "Mismatch between remote svn paths:"
				echo ".ddm : $REPOSITORY_FULL"
				echo "local svn info : $remote"
			fi
		fi
	elif [ "$repotype" == 'vfs' ]
	then
		check_writable_dir "$path" "$repotype dataset" $exitcode
	fi
}


set_dataset_info()
{
	# here we set all the variables, based on what DATASET_LOCAL_REL (given by user) & PWD are
	# DATASET_LOCAL_REL             # identifier to dataset, taken against pwd ( could be '.','..', nothing at all, dirname, full path, ..) 

	# REPOSITORY_FULL=''            # full path to repository (remote or locally if network mount)
	# (REPOSITORY_TYPE)             # type of the repository (not set here)
	# DATASET_LOCAL_FULL=''         # full path to dataset locally (including type suffix if any)
	# DATASET_LOCAL_PARENT_FULL=''  # the path of the parent directory of the dataset ( aka dirname )
	# DATASET_LOCAL_BASE=''         # like $DATASET_LOCAL_FULL but no path ( aka basename) 
	# DATASET_LOCAL_NAME=''         # like $DATASET_LOCAL_BASE but no type suffix
	# DATASET_LOCAL_TYPE=''         # buffer,copy,extensions etc

	#remove trailing /, if any
	DATASET_LOCAL_REL=`sed 's#/$##' <<< $DATASET_LOCAL_REL`

	if [ -z "$DATASET_LOCAL_REL" ] || [ "$DATASET_LOCAL_REL" == '.' ]
	then
		DATASET_LOCAL_FULL=$PWD
	elif [ "$DATASET_LOCAL_REL" == '..' ]
	then
		DATASET_LOCAL_FULL=`dirname $PWD`
	else
		#prepend pwd to make path absolute, if needed
		if grep '^/' <<< $DATASET_LOCAL_REL > /dev/null
		then
			DATASET_LOCAL_FULL=$DATASET_LOCAL_REL
		else
			[ $PWD == '/' ] && PWD=''
			DATASET_LOCAL_FULL=$PWD/$DATASET_LOCAL_REL
		fi
	fi

	DATASET_LOCAL_BASE=`          basename $DATASET_LOCAL_FULL`
	DATASET_LOCAL_PARENT_FULL=`    dirname $DATASET_LOCAL_FULL`

	DATASET_LOCAL_TYPE_NEW=
	DATASET_LOCAL_NAME=$DATASET_LOCAL_BASE
	echo_debug "Trying to filter a dataset type out of the dataset name.  Beginning with: \$DATASET_LOCAL_NAME : $DATASET_LOCAL_NAME  \$DATASET_LOCAL_TYPE_NEW: $DATASET_LOCAL_TYPE_NEW"
	for allowed in $ALLOWED_DATASET_TYPES
	do
		echo_debug "If type still empty, testing $allowed"
		if [[ -z "$DATASET_LOCAL_TYPE_NEW" && "$DATASET_LOCAL_BASE" =~ ${allowed}$ ]]
		then
			echo_debug "Found $allowed !"
			#DATASET_LOCAL_NAME=${DATASET_LOCAL_BASE/-$allowed} # cannot check end of string with this?
			DATASET_LOCAL_NAME=`sed 's/-'$allowed'$//' <<< "$DATASET_LOCAL_BASE"`
			DATASET_LOCAL_TYPE_NEW="${BASH_REMATCH[0]}"
			echo_debug "We assigned this now: \$DATASET_LOCAL_NAME : $DATASET_LOCAL_NAME  \$DATASET_LOCAL_TYPE_NEW: $DATASET_LOCAL_TYPE_NEW"
		fi
	done

	# if we parsed a type from the dataset name, use that (can still be overridden for checkouts !)
	# this behavior is still subject to change/discussion.  let me know if you want this otherwise (TODO: document this better too)
	[ $ACTION != checkout -a -n "$DATASET_LOCAL_TYPE_NEW" ] && DATASET_LOCAL_TYPE=$DATASET_LOCAL_TYPE_NEW
	
	[ "$REPOSITORY_TYPE" == 'svn' ] && REPOSITORY_FULL=`echo $SVN_BASE | sed 's#/$##'`"/${SVN_PREFIX}${DATASET_LOCAL_NAME}${SVN_SUFFIX}"
	[ "$REPOSITORY_TYPE" == 'vfs' ] && REPOSITORY_FULL=`echo $VFS_BASE | sed 's#/$##'`"/${VFS_PREFIX}${DATASET_LOCAL_NAME}${VFS_SUFFIX}"
	
	echo_debug "set_dataset_info finished"
	echo_debug_vars	                                                        
}


 #
 # VALIDATORS.  VALIDATE STUFF
 #

validate_all ()
{
	validate_dataset
	validate_repository
	validate_action
}


validate_dataset ()
{
	# DEFAULT_DATASET_TYPE no validation needed. user can't override
	check_is_in "$DATASET_LOCAL_TYPE" "$ALLOWED_DATASET_TYPES" 'dataset type' 2
	
	if [ $REPOSITORY_TYPE == 'svn' ] && [ $DATASET_LOCAL_TYPE == 'blob' -o $DATASET_LOCAL_TYPE == 'direct' ]
	then
		echo "A $DATASET_LOCAL_TYPE dataset for a svn repository? That wouldn't make much sense now would it?" 2
	fi

}


validate_repository ()
{ 
	check_is_in "$DEFAULT_REPOSITORY_TYPE" "$ALLOWED_REPOSITORY_TYPES" 'repo type' 3
	check_is_in "$REPOSITORY_TYPE"         "$ALLOWED_REPOSITORY_TYPES" 'repo type' 3
}


validate_action ()
{
	if [ -z "$ACTION" ]
	then
		echo_die "You must specify an action with -a <action>.  $ACTION_NAME_HELPSTRING" 4
	fi

	check_is_in "$ACTION" "$ALLOWED_ACTIONS" action 4
}


 #
 # WORKERS. CAN BE CALLED BY CALLBACKS OR OTHER WORKERS.
 #
 
# delete files from dataset

deletefiles ()
{
	deletelist="$1"
	deletelistsize=`grep -cv '^$' <<< "$deletelist"`
	echo_debug "deletelist (size: $deletelistsize) : $deletelist"

	echo_verbose "Deleting old files ... ($deletelistsize items)."
	IFS=$'\n'
	for file in $deletelist
	do
		file=`echo $file | sed 's/ $//'`
		if [ "$file" != '..' ] && [ "$file" != '.' ]
		then
			file=`echo $file | xargs echo` #remove escape characters.. dont know if this is the best way
			wrap_rm "$DATASET_LOCAL_FULL/$file" -r
		fi
	done
}	


# keep files in the dataset

keepfiles ()
{
	keeplist="$1"
	keeplistsize=`grep -cv '^$' <<< "$keeplist"`
	echo_debug "keeplist (size: $keeplistsize) : $keeplist"
	
	echo_verbose "Keeping these files... ($keeplistsize items)."
        IFS=$'\n'
        for file in $keeplist
        do
        	file=`echo "$file" | xargs echo` # -0 fixes quotes but keeps escapes :(
		wrap_keep "$DATASET_LOCAL_FULL/$file"
        done
}


#copy files from repo into dataset

getfiles ()
{
	# note: it's your own responsability to append a '/' if you want to rsync directory contents before calling this function.
	getlist="$1"
	getlistsize=`grep -cv '^$' <<< "$getlist"`
	echo_debug "getlist (size: $getlistsize) : $getlist"
	
	echo_verbose "Copying/syncing files  ... ($getlistsize items)."
	IFS=$'\n'
	for file in $getlist
	do
		wrap_rsync -avuq '.ddm*' "$REPOSITORY_FULL/$file" "$DATASET_LOCAL_FULL/$file"
	done
}


no_action ()
{
	echo "No action taken.  This is either not implemented yet or just wouldn't make much sense."
	echo "You can 1) implement what you want in $DATASET_LOCAL_FULL/.ddm"
	echo "        2) file a feature request if this seems reasonable (or send a patch)"
	if [ -n "$1" ]
	then
		echo "Note from the author: $1"
	fi
}


wrap_keep ()
{
	# $1 filename
	if [ -z "$1" ]
	then
		echo_die 'internal error: insufficient parameters to keep wrapper' 100
	fi
	
	file=$1
	echo_verbose " * $file"
	echo_pretend "not touching $file"
}


wrap_mkdir ()
{
	# $1 path
	if [ -z "$1" ]
	then
	echo_die 'internal error: insufficient parameters to mkdir wrapper' 100
	fi
	dir=$1
	echo_pretend_verbose -n "Mkdir'ing $dir ... "
	if [ "$PRETEND" == '0' ]
	then
		mkdir -p "$dir"
		if [ $? -eq 0 ]
		then
			echo_verbose -n "success"
		else
			echo -n "failed" >&2
		fi
	else
		#TODO: dirs maybe need to be created recursively.  we should go up the path until the first thing that exists, and test that...
		true
		#parent=`dirname $dir`
		#if [ -d "$parent" -a -w "$parent" -a -x "$parent" ]
		#then
		#	echo_verbose -n " should work"
		#else
		#	echo_verbose -n " will fail"
		#fi
	fi
	echo
}


wrap_cd ()
{
	# 1$ path
	if [ -z "$1" ]
	then
		echo_die 'internal error: insufficient parameters to cd wrapper' 100
	fi

	path=$1
	echo_pretend_verbose -n "Cd'ing into $path ... "
	if [ "$PRETEND" == '0' ]
	then
		cd $path
		if [ $? -eq 0 ]
		then
			echo_verbose -n "success"
		else
			echo -n "failed" >&2
		fi
	else
		if [ -d "$path" -a -r "$path" -a -x "$path" ]
		then
			echo_verbose -n " should work"
		else
			echo_verbose -n " will fail"
		fi
	fi
	echo
}

wrap_rm ()
{
	# $1 filename
	# $2 options
	if [ -z "$1" ]
	then
		echo_die 'internal error: insufficient parameters to rm wrapper' 100
	fi

	file=$1
	
	echo_verbose -n " * $file... "

	if [ -e "$file" ]
	then
		if [ "$PRETEND" == '0' ]
		then
			echo_verbose -n "found. removing... "
			rm "$2" "$file"
			if [ $? -eq 0 ]
			then
				echo_verbose "success"
			else
				echo_verbose "failed"
			fi
		else
			echo_verbose -n "found. pretending remove... "
			echo_pretend -n "rm $2 $file"
			if [ -w "$file" ] && [[ ! -d "$file" || "$2" = *r* ]]
			then
				echo_verbose -n " should work"
			else
				echo_verbose -n " will fail (file not writable)"
			fi
			echo
		fi
	else
		echo_verbose "not found"
	fi
}

	
wrap_svn ()
{
	# $1 svn action (update,commit,..)
	# $@ all svn args

	if [ -z "$1" ]
	then
		echo_die 'internal error: insufficient parameters to svn wrapper' 100
	fi
	if [ "$PRETEND" == '1' -a "$1" != info ] #svn info doesn't need to be pretended.  we can (and should) safely do this.
	then
		echo_pretend "svn $@"
		[ "$1" == 'checkout' ] && echo_pretend "svn propedit svn:ignore . --non-interactive --editor-cmd 'echo .ddm >>'"
	else
		echo_verbose " * svn $@ ..."
		if svn $@
		then
			echo "success"
			if [ "$1" == 'checkout' ]
			then
				echo_verbose " * svn setting ignore for .ddm ..."
				#TODO: paths??
				if svn propedit svn:ignore . --non-interactive --editor-cmd 'echo .ddm >>'
				then
					echo "success"
				else
					echo "failed"
				fi
			fi
		else
			echo "failed"
		fi
	fi
}


wrap_rsync ()
{
	# $1 options (eg -avu)
	# $2 ignore pattern (eg '.ddm*')
	# $3 source
	# $4 destination
	# $5 1 -> ask user for --delete (optional)

	if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] || [ -z "$4" ]
	then
		echo_die 'internal error: insufficient parameters to rsync wrapper' 100
	fi
	
	opts=$1
	ignore=$2
	src=$3
	dest=$4
	ask_del=$5
	
	# if source is a directory, append / if needed. (needed for rsync)
	if [ -d "$src" ] && ! grep '/$' >/dev/null <<< "$src"
	then
		src="$src/"
	fi
	
	# if ddm is in verbose mode, add -v flag for rsync
	if [ "$VERBOSE" == '1' ]
	then
		extra_opts="-v"
	else
		extra_opts=''
	fi
	#NOTE: rsync manpage uses both --exclude='' and --exclude '' forms.  I guess both work
	user_response=
	if [ "$ask_del" == '1' ]
	then
		ask_user 'delete extraneous files from dest dirs? (rsync --delete)' 1
	fi

	echo_verbose -n " * $src ... "
	if [ "$user_response" == 'y' ]
	then
		[ "$PRETEND" == '1' ] && echo_pretend "rsync $opts $extra_opts --delete --exclude $ignore $src $dest"
		[ "$PRETEND" == '0' ] &&               rsync $opts $extra_opts --delete --exclude "$ignore" "$src" "$dest"
	else
		[ "$PRETEND" == '1' ] && echo_pretend "rsync $opts $extra_opts --exclude $ignore $src $dest"
		[ "$PRETEND" == '0' ] &&               rsync $opts $extra_opts --exclude "$ignore" "$src" "$dest"
	fi
	result=$?
	[ $result -eq 0 -a "$PRETEND" == '0' ] && echo_verbose "success"
	[ $result -gt 0 -a "$PRETEND" == '0' ] && echo "failed"
}


# calls a certain callback, if it exists

do_callback ()
{
	# $1 callback name
	
	if [ -z "$1" ]
	then
		echo_die 'internal error: insufficient parameters in do_callback' 100
	fi

	if type -t $1 | grep function >/dev/null
	then
		echo_verbose "callback $1 defined. executing..."
		PWD_BACKUP=`pwd`
		$1
		cd $PWD_BACKUP
	else
		echo_verbose "callback $1 not defined. skipping."
	fi

}


 #
 # CALLBACKS. CAN BE OVERWRITTEN IN DATASET-SPECIFIC .ddm FILES. there you can even ignore the $DATASET_LOCAL_TYPE and REPOSITORY_TYPE if you want
 #

prebackup ()
{
	if [ $DATASET_LOCAL_TYPE == "extension" ]
	then
		DATASET_LOCAL_REL=${DATASET_LOCAL_FULL}
		REPOSITORY_FULL=${REPOSITORY_FULL}-extension
		DATASET_LOCAL_NAME=`basename ${DATASET_LOCAL_FULL}`
		DATASET_LOCAL_TYPE="copy"
		check_repo_path "$REPOSITORY_FULL" "$REPOSITORY_TYPE"
	fi

	do_callback precommit
}


dobackup ()
{
	do_callback docommit
}


postbackup ()
{
	do_callback postcommit
}


docheckout ()
{
	# all datatypes: blob buffer cache copy direct extension selection
	# defined for: blob, copy, direct/vfs, selection

	if [ "$REPOSITORY_TYPE" == 'svn' ] && [ $DATASET_LOCAL_TYPE == "buffer" ]
	then
		no_action "maybe we could copy (only) the remote directories as template hierachy? Is this what you expected?"
	elif [ "$REPOSITORY_TYPE" == 'svn' ] && [ $DATASET_LOCAL_TYPE == "extension" ]
	then
		no_action "maybe we could copy (only) the remote directories as template hierachy? Is this what you expected?"
	elif [ "$REPOSITORY_TYPE" == 'svn' ] && [ $DATASET_LOCAL_TYPE == "direct" ]
	then
		no_action "you want a mountpoint to the svn repository itself?  seems a bit far fetched..."
	elif [ "$REPOSITORY_TYPE" == 'svn' ] && [ $DATASET_LOCAL_TYPE == "selection" ]
	then
		echo_verbose "You are checking out a selection: i assume you will delete what you don't need afterwards?"
		wrap_svn checkout $REPOSITORY_FULL $DATASET_LOCAL_FULL
	elif [ "$REPOSITORY_TYPE" == 'svn' ] && [ $DATASET_LOCAL_TYPE == "copy" -o $DATASET_LOCAL_TYPE == "blob" ]
	then
		wrap_svn checkout $REPOSITORY_FULL $DATASET_LOCAL_FULL
	elif [ "$REPOSITORY_TYPE" == 'vfs' ] && [ $DATASET_LOCAL_TYPE == "selection" ]
	then
		echo_verbose "You are checking out a selection: either you delete what you don't need afterwards, or you override this function to do what you want"
		wrap_rsync -au '.ddm*' "$REPOSITORY_FULL/" "$DATASET_LOCAL_FULL"	
	elif [ "$REPOSITORY_TYPE" == 'vfs' ] && [ $DATASET_LOCAL_TYPE == "copy" -o $DATASET_LOCAL_TYPE == "blob"  ] 
	then
		wrap_rsync -au '.ddm*' "$REPOSITORY_FULL/" "$DATASET_LOCAL_FULL" 
	elif [ $DATASET_LOCAL_TYPE == "direct" ]
	then
		no_action "TODO: check for mountpoints, maybe parents and try to mount"
	else
		no_action
	fi
}


docommit ()
{
	# all datatypes: blob buffer cache copy direct extension selection
	#defined for: blob, buffer/vfs (alias for flush by default), copy

	if [ "$REPOSITORY_TYPE" == 'svn' ] && [ $DATASET_LOCAL_TYPE == "blob" -o $DATASET_LOCAL_TYPE == "copy" -o $DATASET_LOCAL_TYPE == "selection" ]
	then
		wrap_svn commit $DATASET_LOCAL_FULL -m "$MESSAGE"
	elif [ "$REPOSITORY_TYPE" == 'svn' ]
	then
		no_action
	fi
	if [ "$REPOSITORY_TYPE" == 'vfs' ]
	then
		if [ $DATASET_LOCAL_TYPE == "copy" -o $DATASET_LOCAL_TYPE == "blob" ]
		then
			wrap_rsync -au '.ddm*' "$DATASET_LOCAL_FULL/" "$REPOSITORY_FULL" 1 
		elif [ $DATASET_LOCAL_TYPE == "buffer" ]
		then
			do_callback doflush
		elif [ $DATASET_LOCAL_TYPE == "selection" ]
		then
			no_action "Maybe the user made changes to the small subset of data he has and wants to commit those..
			          gonna be hard to implement something like that..."
		elif [ $DATASET_LOCAL_TYPE == "extension" ]
		then
			no_action "Since this is an extension, i woulnd't know where to send data to...did you mean backup?"
		elif [ $DATASET_LOCAL_TYPE == "direct" ]
		then
			no_action "Maybe we could umount? ( if it's a direct mount) and ask to umount if the parent is a mountpoint"
		elif [ $DATASET_LOCAL_TYPE == "cache" ]
		then
			no_action "You want to commit a cache? doesn't make much sense to me..."
		fi

	fi 

}


doflush ()
{
	if [ $DATASET_LOCAL_TYPE == "buffer" ]
	then
		#NOTE: --remove-source-files only removes files, not directories. this can be good (directories define structure) or bad (directories function as data)
		wrap_rsync '-au --remove-source-files' '.ddm*' "$DATASET_LOCAL_FULL/" "$REPOSITORY_FULL" 0
		failures=`find $DATASET_LOCAL_FULL -type f | grep -cv "$DATASET_LOCAL_FULL/.ddm"`
		echo "Warning: still $failures un(successfully) flushed files in the buffer" >&2
		echo "Successfully transfered files deleted. Directories are kept; you must delete them yourself if you don't need them anymore"
	elif [ $DATASET_LOCAL_TYPE == "cache" ]
	then
		deletefiles "`ls -1 $DATASET_LOCAL_FULL/`"
	else
		no_action
	fi
}


prerestore ()
{
	do_callback preupdate
}


dorestore ()
{
	# all datatypes: blob buffer cache copy direct extension selection
	do_callback doupdate
	if [ "$REPOSITORY_TYPE" == 'svn' ]
	then
		wrap_svn revert -R $DATASET_LOCAL_FULL
		#TODO: we could still have conflicts or removed directories
		do_callback doupdate
	fi
}


postrestore ()
{
	do_callback postupdate
}


doupdate ()
{
	# all datatypes: blob buffer cache copy direct extension selection
	# defined for: all svn except direct, selection/vfs, copy/vfs, blob/vfs, direct/vfs (alias for checkout)
	if [ "$REPOSITORY_TYPE" == 'svn' -a ! $DATASET_LOCAL_TYPE == "direct" ]
	then
		wrap_svn update $DATASET_LOCAL_FULL
	elif [ "$REPOSITORY_TYPE" == 'vfs' ] && [ $DATASET_LOCAL_TYPE == "selection" ]
	then
		echo_verbose "For now I just delete everything in the dataset and fetch the new contents."
		echo_verbose "You can override this with your own functions (see documentation)"
		deletefiles "`ls -1 $DATASET_LOCAL_FULL`"
		getfiles "`ls -1 $REPOSITORY_FULL`"
	elif [ "$REPOSITORY_TYPE" == 'vfs' ] && [ $DATASET_LOCAL_TYPE == "copy" -o $DATASET_LOCAL_TYPE == "blob" ]
	then	
		wrap_rsync -au '.ddm*' $REPOSITORY_FULL/ $DATASET_LOCAL_FULL
	elif [ $DATASET_LOCAL_TYPE == "direct" ]
	then
		do_callback docheckout
	else
		no_action
	fi
}


 #
 # START THE ACTUAL PROCESSING
 #

# load ddm plugins according to the XDG basedir spec
PWD_BEFORE=`pwd`
for data_dir in `echo ${XDG_DATA_HOME:-$HOME/.local/share}:${XDG_DATA_DIRS:-'/usr/local/share/:/usr/share/'} | sed 's/:/\t/g'`
do
	data_dir=`echo $data_dir | sed 's#/$##'`
	if [ -r $data_dir/ddm/plugins ]
	then
		for plugin in $data_dir/ddm/plugins/*.sh
		do
			echo_verbose_buffer="${echo_verbose_buffer}Plugin   $plugin loading...\n"
			source "$plugin"
			if [ $? -gt 0 ]
			then
				echo "Warning: $plugin loading failed"
			else
				echo_verbose_buffer="${echo_verbose_buffer}Plugin   $plugin loaded\n"
			fi
			cd $PWD_BEFORE #undo any changes to PWD the plugin might have done
		done
	fi
done

#VARS OVERRIDABLE BY USER

SVN_BASE=''
SVN_PREFIX=''
SVN_SUFFIX=''
VFS_BASE=''
VFS_PREFIX=''
VFS_SUFFIX=''

DEFAULT_REPOSITORY_TYPE='vfs'
ALLOWED_REPOSITORY_TYPES='vfs svn'
ALLOWED_DATASET_TYPES='blob buffer cache copy direct extension selection'
ALLOWED_ACTIONS='backup checkout commit flush restore update'
IGNORE_DATASET_REMOTE_SVN=0

# look for a ddmrc to load according to the XDG basedir spec

for config_dir in `echo ${XDG_CONFIG_HOME:-$HOME/.config}:${XDG_CONFIG_DIRS:-/etc/xdg} | sed 's/:/\t/g'`
do
	config_dir=`sed 's#/$##' <<< $config_dir`
	if [ -r $config_dir/ddm/ddmrc ]
	then
		source $config_dir/ddm/ddmrc
		ddmrc_included='y'
		if [ $? -gt 0 ]
		then
			echo_die "There is a problem with your $config_dir/ddm/ddmrc . probably incorrect syntax" 1
		else
			echo_verbose_buffer="${echo_verbose_buffer}Successfully included $config_dir/ddm/ddmrc\n"
		fi
		break
	fi
done
if [ -z "$ddmrc_included" ]
then
	echo_verbose_buffer="${echo_verbose_buffer}Something failed while including ddmrc.  If you want to override some values, create a ${XDG_CONFIG_HOME:-$HOME/.config}/ddm/ddmrc in valid bash\n"
fi

PWD_AFTER=`pwd`
[ "$PWD_AFTER" != "$PWD_BEFORE" ] && echo_die "Please don't cd in your ddmrc script.  PWD must remain the same!" 1


#VARS NOT OVERRIDABLE BY USER
PWD=$PWD_BEFORE

DEFAULT_DATASET_TYPE='copy' # i dont think this should be changed by the user
ACTION_HELPSTRING="(one of: $ALLOWED_ACTIONS)"
DATASET_NAME_HELPSTRING="(pointer to/name of directory, default is \$PWD ($PWD))"
DATASET_TYPE_HELPSTRING="(one of: $ALLOWED_DATASET_TYPES, default is $DEFAULT_DATASET_TYPE). Usually only needed explicitly for checkouts"
REPOSITORY_TYPE_HELPSTRING="(one of: $ALLOWED_REPOSITORY_TYPES, default is $DEFAULT_REPOSITORY_TYPE)"
MESSAGE_HELPSTRING="(usually only used for commits in svn)"
	

DATASET_LOCAL_REL=$PWD
DATASET_LOCAL_TYPE=$DEFAULT_DATASET_TYPE
REPOSITORY_TYPE=$DEFAULT_REPOSITORY_TYPE
MESSAGE=''
PRETEND=0

validate_repository

while getopts “:a:d:t:r:m:vhp” OPTION
do
	case $OPTION in
	a)
		if [ -z "$OPTARG" ]
		then
			echo_die "To use -a, specify an action as argument.  $ACTION_HELPSTRING" 5
		fi
		ACTION=$OPTARG
		validate_action
		;;
	d)
		if [ -z "$OPTARG" ]
		then
			echo_die "To use -d, specify a dataset as argument.  $DATASET_NAME_HELPSTRING" 5
		fi
		DATASET_LOCAL_REL=$OPTARG
		;;
	t)
		if [ -z "$OPTARG" ]
		then
			echo_die "To use -t, specify a dataset-type as argument.  $DATASET_TYPE_HELPSTRING" 5
		fi
		DATASET_LOCAL_TYPE=$OPTARG
		validate_dataset
		;;
	r)
		if [ -z "$OPTARG" ]
		then
			echo_die "To use -r, specify a repo-type as argument.  $REPOSITORY_TYPE_HELPSTRING" 5
		fi
		REPOSITORY_TYPE=$OPTARG
		validate_repository
		;;
	m)
		if [ -z "$OPTARG" ]
		then
			echo_die "To use -m, specify a message as argument.  $MESSAGE_HELPSTRING" 5
		fi
		MESSAGE=$OPTARG
		;;
	p)
		PRETEND=1
		;;
	v)
		VERBOSE=1
		;;
	h)
		usage
		exit 0
		;;
	?)
		usage
		exit 5
		;;
	esac
done

#We now know if we want verbose or not, so we can flush pending output from before we parsed the arguments
echo_verbose $echo_verbose_buffer

echo_debug "Validate_all after arguments processing"
validate_all

set_dataset_info

# override some stuff in the .ddm script in the dataset

if [ -r $DATASET_LOCAL_FULL/.ddm ]
then
	source $DATASET_LOCAL_FULL/.ddm
	if [ $? -ne 0 ]
	then
		echo_die "Failed sourcing $DATASET_LOCAL_FULL/.ddm" 2
	else
		echo_verbose "Included $DATASET_LOCAL_FULL/.ddm" 
	fi
fi

echo_debug_vars
echo_debug "Validate_all after set_dataset_info + .ddm loading"
validate_all

PWD_AFTER=`pwd`
[ "$PWD_AFTER" != "$PWD" ] && echo_die "Please don't cd in your .ddm scripts.  PWD must remain the same!" 2   
        

check_repo_path "$REPOSITORY_FULL" "$REPOSITORY_TYPE"

if [ "$ACTION" != checkout ]
then
	check_dataset_path "$DATASET_LOCAL_FULL" "$REPOSITORY_TYPE"
else
	check_writable_dir `dirname "$DATASET_LOCAL_FULL"` "parent dir for $REPOSITORY_TYPE dataset" 2
fi	

echo_verbose "action $ACTION on dataset $DATASET_LOCAL_FULL ( $DATASET_LOCAL_TYPE ), repo $REPOSITORY_FULL ( $REPOSITORY_TYPE ) [ DEBUG $DEBUG - PRETEND $PRETEND - VERBOSE $VERBOSE ]"

if [ "$PRETEND" == '1' ]
then
	echo_verbose "Warning: do not just copy paste the given commands into a shell.  your shell will interpret and glob them and they might not work as expected.  just remove the -p flag to execute."
fi

for prefix in pre 'do' post
do
	do_callback $prefix$ACTION
done

echo_pretend "Done"