jstart

#!/usr/local/bin/cbsd
#v10.1.6
MYARG=""
MYOPTARG="jname inter"
MYDESC="Start jail"
ADDHELP="inter=0 to prevent any questions and to accept answers by default\n"
CBSDMODULE="jail"
EXTHELP="wf_jstop_jstart.html"

. ${subr}
. ${system}
. ${strings}
. ${tools}
. ${workdir}/universe.subr
readconf buildworld.conf
readconf jail-freebsd-default.conf

[ -z "${1}" ] && select_jail_by_list -s "List of offline jail" -a "Off" -r ${sqlreplica}
init $*

# users custom stop script
# $1 - is source dir
external_exec_script()
{
	local _res
	local _dir="${1}"

	[ -z "${_dir}" ] && return 1

	[ ! -d "${jailsysdir}/${jname}/${_dir}" ] && return 0
	[ -z "$( /bin/ls ${jailsysdir}/${jname}/${_dir}/ )" ] && return 0
	[ -d "${data}/tmp/${_dir}" ] && rm -rf "${data}/tmp/${_dir}"
	/bin/cp -a "${jailsysdir}/${jname}/${_dir}" ${data}/tmp/${_dir}

	/usr/bin/find "${data}/tmp/${_dir}" -type f -depth 1 -maxdepth 1 -exec /usr/bin/basename {} \; | while read _file; do
		${ECHO} "${MAGENTA}Execute script: ${GREEN}${_file}${NORMAL}"
		jexec jname="${jname}" /tmp/${_dir}/${_file}
	done
	rm -rf "${data}/tmp/${_dir}"
}

# users custom stop for executing in master host
# $1 - is source dir name ( master_poststart.d master_prestart.d )
external_exec_master_script()
{
	local _res
	local _dir="${1}"

	[ -z "${_dir}" ] && return 1

	local _srcdir="${jailsysdir}/${jname}/${_dir}"

	[ ! -d "${_srcdir}" ] && return 0
	[ -z "$( /bin/ls ${_srcdir}/ )" ] && return 0

	/usr/bin/find "${_srcdir}" -type f -depth 1 -maxdepth 1 -exec /usr/bin/basename {} \; |while read _file; do
		${ECHO} "${MAGENTA}Execute master script: ${GREEN}${_file}${NORMAL}"
		${_srcdir}/${_file}
	done
}

# check environment scripts from tests dir
# when error code != 0 - exit
# $1 - is source dir name ( tests )
check_environment_script()
{
	local _res
	local _dir="${1}"
	local _testfiles

	[ -z "${_dir}" ] && return 1

	local _srcdir="${jailsysdir}/${jname}/${_dir}"

	[ ! -d "${_srcdir}" ] && return 0
	[ -z "$( /bin/ls ${_srcdir}/ )" ] && return 0

	_testfiles=$( /usr/bin/find "${_srcdir}" -type f -depth 1 -maxdepth 1 -exec /usr/bin/basename {} \; )

	for _file in ${_testfiles}; do
		${ECHO} "${MAGENTA}Check environment script: ${GREEN}${_file}${NORMAL}"
		env jname=${jname} ${_srcdir}/${_file}
		[ $? -ne 0 ] && err 1 "${MAGENTA}Error: ${GREEN}${_file}${NORMAL}"
	done
}



exec_master_poststart()
{
	eval CMD=\${exec_master_poststart}
	[ -z "${CMD}" -o "${CMD}" = "0" ] && return 0
	${ECHO} "${MAGENTA}${jname} Master exec afterstart: ${GREEN}${CMD}${NORMAL}"
	${CMD}
}

exec_poststart()
{
	eval CMD=\${exec_poststart}
	[ -z "${CMD}" -o "${CMD}" = "0" ] && return 0
	${ECHO} "${MAGENTA}${jname} exec exec_poststart: ${GREEN}${CMD}${NORMAL}"
	jexec jname=${jname} ${CMD}
}

exec_cbsdjail_first_boot()
{
	if [ -f ${path}/etc/rc.cbsdjail_first_boot ]; then
		jexec jname=${jname} /bin/sh /etc/rc.cbsdjail_first_boot
		rm -f ${path}/etc/rc.cbsdjail_first_boot
	fi
}

exec_master_prestart()
{
	eval CMD=\${exec_master_prestart}
	[ -z "${CMD}" -o "${CMD}" = "0" ] && return 0
	${ECHO} "${MAGENTA}${jname} master exec prestart: ${GREEN}${CMD}${NORMAL}"
	${CMD}
}

exec_prestart()
{
	eval CMD=\${exec_prestart}
	[ -z "${CMD}" -o "${CMD}" = "0" ] && return 0
	${ECHO} "${MAGENTA}${jname} exec exec_prestart: ${GREEN}${CMD}${NORMAL}"
	jexec jname=${jname} ${CMD}
}

# return in $FWNUM first free ipfw num
# return 0 if not available
get_first_available_fwcount()
{

	for i in $( seq ${fwcount_st} ${fwcount_end} ); do
		/sbin/ipfw -q show ${i} > /dev/null 2>&1
		if [ $? -ne 0 ]; then
			FWNUM=$i
			return 0
		fi
	done

	[ $i -eq ${fwcount_end} ] && FWNUM=0
}

fwcounter()
{
	[ ${myjid} -eq 0 ] && return 0
	[ "${ipfw_enable}" = "0" -a -z "`sysctl -n net.inet.ip.fw.enable 2>/dev/null`" ] && return 0
	#FWIN
	get_first_available_fwcount

	if [ $FWNUM -eq 0 ]; then
		${ECHO} "${MAGENTA}Warning: FW counter not available in ${fwcount_st}-${fwcount_end} range. Skip${NORMAL}"
		return 0
	fi

	${ECHO} "${MAGENTA}Setup: ${GREEN}FW counter${NORMAL}"
	/sbin/ipfw -q add ${FWNUM} count ip from any to me jail ${myjid} && echo ${FWNUM} > ${ftmpdir}/${jname}-fwin

	#FWOUT
	get_first_available_fwcount

	if [ $FWNUM -eq 0 ]; then
		${ECHO} "${MAGENTA}Warning: FW counter not available in ${fwcount_st}-${fwcount_end} range. Skip${NORMAL}"
		return 0
	fi

	/sbin/ipfw -q add ${FWNUM} count ip from me to any jail ${myjid} && echo ${FWNUM} > ${ftmpdir}/${jname}-fwout
}


# check that the emulator is able to execute commands
emulator_exec_check()
{
	local _res

	[ -z "${emulator}" ] && return 0
	[ "${emulator}" = "bhyve" ] && return 0
	[ "${emulator}" = "jail" ] && return 0
	[ "${emulator}" = "0" ] && return 0
	_res=$( /usr/sbin/chroot ${path} /bin/${emulator} ${path}/bin/sh -n "echo ping" 2>&1 )
	[ $? -ne 0 ] && printf "${_res}"
}


start_bhyve()
{
	#    echo "sh /usr/share/examples/bhyve/vmrun.sh -d ${MDFILE} -l com1,/dev/${jname} ${jname}"

	${ECHO} "${MAGENTA}For bhyve mode use: ${GREEN}cbsd bstart jname=${jname} ${MAGENTA}instead.${NORMAL}"
	exit 0
}

# chase 'grep -A28 ^20140930: /usr/ports/CHANGES'
set_changes_20140930()
{
	local _tpl="CBSD autotpl for CHANGES 20140930"

	[ ${applytpl} -eq 0 ] && return 0
	[ "${ver}" = "empty" ] && return 0

	local nodever=$( ${miscdir}/elf_tables --freebsdver /bin/sh 2>/dev/null )
	local jailver=$( ${miscdir}/elf_tables --freebsdver ${path}/bin/sh 2>/dev/null )
	local osversion=$( ${miscdir}/elf_tables --ver ${path}/bin/sh 2>/dev/null )

	[ "${nodever}" = "${jailver}" ] && return 0

	${ECHO} "${MAGENTA}Applying /usr/ports/CHANGES -A28 ^20140930 work-around${NORMAL}"
	# we need put fake UNAME_r-stuff environment for correct works of port system
	if ! is_number ${ver}; then
		# is stable
		local release="${ver}.0-STABLE"
	else
		# is release
		local release="${ver}-RELEASE"
	fi

	# make backup and add records
	[ -f "${data}/etc/make.conf" ] && cp ${data}/etc/make.conf ${data}/etc/make.conf.bak

	cat >> ${data}/etc/make.conf <<EOF

OSVERSION+= ${osversion}			# ${_tpl}
UNAME_ENV+= OSVERSION=${osversion}		# ${_tpl}
UNAME_ENV+= UNAME_s=FreeBSD			# ${_tpl}
UNAME_ENV+= UNAME_r=${release}			# ${_tpl}
UNAME_ENV+= UNAME_v="\${UNAME_s} \${UNAME_r}"	# ${_tpl}
.MAKEFLAGS: \${UNAME_ENV}			# ${_tpl}
MAKE_ENV+= \${UNAME_ENV}			# ${_tpl}
CONFIGURE_ENV+= \${UNAME_ENV}			# ${_tpl}
SCRIPTS_ENV+= \${UNAME_ENV}			# ${_tpl}

EOF

	cat >> ${data}/etc/profile << EOF

OSVERSION=${osversion}; export OSVERSION		# ${_tpl}
UNAME_s=FreeBSD; export UNAME_s				# ${_tpl}
UNAME_r=${release}; export UNAME_v			# ${_tpl}
UNAME_v="\${UNAME_s} \${UNAME_r}"; export UNAME_v	# ${_tpl}

EOF

	cat >> ${data}/etc/csh.cshrc << EOF

setenv OSVERSION ${osversion}			# ${_tpl}
setenv UNAME_s FreeBSD				# ${_tpl}
setenv UNAME_r ${release}			# ${_tpl}
setenv UNAME_v "\${UNAME_s} \${UNAME_r}"	# ${_tpl}

EOF
}

unset_changes_20140930()
{
	local _tpl="CBSD autotpl for CHANGES 20140930"

	if ${GREP_CMD} "${_tpl}" ${data}/etc/make.conf >/dev/null 2>&1; then
		/bin/cp -a ${data}/etc/make.conf ${data}/etc/make.conf.bak
		${GREP_CMD} -v "${_tpl}" ${data}/etc/make.conf.bak | ${GREP_CMD} "." > ${data}/etc/make.conf
	fi

	if ${GREP_CMD} "${_tpl}" ${data}/etc/profile >/dev/null 2>&1; then
		/bin/cp -a ${data}/etc/profile ${data}/etc/profile.bak
		${GREP_CMD} -v "${_tpl}" ${data}/etc/profile.bak | ${GREP_CMD} "." > ${data}/etc/profile
	fi

	if ${GREP_CMD} "${_tpl}" ${data}/etc/csh.cshrc >/dev/null 2>&1; then
		/bin/cp -a ${data}/etc/csh.cshrc ${data}/etc/csh.cshrc.bak
		${GREP_CMD} -v "${_tpl}" ${data}/etc/csh.cshrc.bak | ${GREP_CMD} "." > ${data}/etc/csh.cshrc
	fi
}



# MAIN for multiple jails
if [ $# -gt 1 -a -z "${jname}" ]; then
	# multiple astart always non interactive
	export inter=0
	# recursive
	JLIST=$*
	for jname in ${JLIST}; do
		[ "${jname}" = "inter=0" ] && continue
		/usr/sbin/daemon -p ${ftmpdir}/jstart.${jname}.$$ /usr/local/bin/cbsd jstart inter=0 jname=${jname}
		#lets save .pid file
		sleep 1
		[ -f "${ftmpdir}/jstart.${jname}.$$" ] && cbsd_pwait --pid=$( /bin/cat ${ftmpdir}/jstart.${jname}.$$ ) --timeout=${parallel}
	done

	wait_for_fpid start ${parallel}

	err 0 "${MAGENTA}Multiple jstart: ${GREEN}done${NORMAL}"
fi


# MAIN
[ -z "$jname" ] && jname=$1
. ${jrcconf}
if [ $? -eq 1 ]; then
	[ $sqlreplica -eq 0 ] && err 1 "${MAGENTA}No such jail: ${GREEN}${jname}${NORMAL}"
	remotenode=$( jwhereis $jname )
	[ -z "${remotenode}" ] && err 1 "${MAGENTA}No such jail: ${GREEN}${jname}${NORMAL}"
	for i in ${remotenode}; do
		if [ "${i}" = "${nodename}" ]; then
			${ECHO} "${MAGENTA}Remote jstart: found on nodename ${GREEN}${nodename}${MAGENTA}. Skipped${NORMAL}"
			continue
		fi
		${ECHO} "${MAGENTA}Remote jstart: ${GREEN}${jname} ${MAGENTA}on${GREEN} ${i}${NORMAL}"
		rexe node=${i} cbsd jstart jname=${jname}
	done
	exit 0
fi

[ ${status} -eq 2 ] && err 1 "${MAGENTA}Jail in slave mode. Please ${GREEN}cbsd jswmode mode=master${MAGENTA} first${NORMAL}"
[ ${status} -eq 3 ] && err 1 "${MAGENTA}Jail in maintenance mode${NORMAL}"
[ ${jid} -ne 0 ] && err 1 "${MAGENTA}Jail ${jname} already running, jid: ${GREEN}${jid}${NORMAL}"
[ "${vnet}" = "1" -a "${vimage_feature}" = "0" ] && err 1 "${MAGENTA}Jail ${GREEN}${jname}${MAGENTA} have vnet=1 flags but your kernel is not support VIMAGE${NORMAL}"
[ "${emulator}" = "bhyve" -a -z "${mdsize}" ] && err 1 "${MAGENTA}Bhyve required for file image${NORMAL}"

TRAP=""

#Check for shared lock
jaillock="${jailsysdir}/${jname}/locked"
if [ -f "${jaillock}" ]; then
	masterhost=$( /bin/cat ${jaillock} )
	if [ "${masterhost}" = "${nodename}" ]; then
		${ECHO} "${MAGENTA}Remove my stale lock file: ${GREEN}${jaillock}${NORMAL}"
		rm -f ${jaillock}
	else
		cur_time=$( /bin/date +%s )
		eval $( /usr/bin/stat -s ${jaillock} )
		difftime=$(( ( cur_time - st_mtime ) / 60 ))

		# 30 minutes outdated lock
		if [ ${difftime} -gt 30 ]; then
			${ECHO} "${MAGENTA}Jail ${GREEN}${jname}${MAGENTA} locked by ${GREEN}${masterhost}${MAGENTA} node${NORMAL}"
			${ECHO} "${MAGENTA}But lock age is too old: ${difftime} min. Removing!!!${NORMAL}"
			/bin/rm -f ${jaillock}
		else
			# still fresh
			$ECHO "${MAGENTA}Jail ${GREEN}${jname}${MAGENTA} locked by ${GREEN}${masterhost}${MAGENTA} node, lock age: ${difftime} min.${NORMAL}"
			err 1 "${MAGENTA}You may remove the lockfile if you believe that jail is not running on this node: ${GREEN}rm -f ${jaillock}${NORMAL}"
		fi
	fi
fi

TRAP="${TRAP} rm -f ${jaillock};"
trap "${TRAP}" HUP INT ABRT BUS TERM EXIT

#Check for md vnode backend
if [ -n "${mdsize}" -a "${mdsize}" != "0" ]; then
	MDFILE="${jailsysdir}/${jname}/image.dat"
	[ ! -f "${MDFILE}" ] && ${ECHO} "${MAGENTA}No such ${MDFILE} but mdsize flags is not null. Skip${NORMAL}" && continue
fi

[ "${emulator}" = "bhyve" ] && start_bhyve

init_target_arch
init_basedir

jcleanup jname=${jname}
[ ! -d "${path}" ] && /bin/mkdir ${path}

check_environment_script "tests"

case ${ip4_addr} in
	[Dd][Hh][Cc][PP])
		ip4_addr=$( dhcpd )
		jset jname=${jname} ip4_addr="${ip4_addr}"
		;;
esac

geniplist ${ip4_addr}

if [ -n "${interface}" -a "${interface}" != "0" -a "${vnet}" -eq "0" ]; then
	### CHECK FOR IP ####
	for ips in ${IPS}; do
		iptype "${ips}" ||true
		[ -z "${IWM}" ] && continue
		[ -n "${VHID}" ] && continue
		#prevent to use nodeip
		[ "${IWM}" = "${nodeip}" ] && err 1 "${MAGENTA}Error: Jail can not take nodeip when interface is not equal 0: ${GREEN}${nodeip}${NORMAL}"
		checkip ip=${IWM} check=1
		IPUSE=$?
		case ${IPUSE} in
			0)
				${ECHO} "${MAGENTA}ip ${IWM} not in pool range${NORMAL}"
				continue
			;;
			2)
				${ECHO} "${MAGENTA}Ip ${IWM} already exists in LAN${NORMAL}"
				continue
			;;
		esac
	done
fi

if [ "$vnet" = "1" ]; then
	# for vnet we can make another action
	. ${vimageconf}
	# when ip=0 and interface=auto we must use default interface for upstream
	[ "${interface}" = "0" -o "${interface}" = "auto" ] && interface=$( cbsd getnics-by-ip ip=0.0.0.0 )

	if ! mybridge=$( get_my_device bridge ${interface} ); then
		err 1 "${MAGENTA}Error: Cant get_my_device for ${interface}: ${mybridge}${NORMAL}"
	fi
	myepair=$( get_my_epair ${mybridge} )
	[ $? -eq 1 ] && err 1 "${MAGENTA}Error: Cant get_my_epair by: ${GREEN}${mybridge}${NORMAL}"
	ifconfig ${myepair}a description ${jname} up
#	dhclient ${myepair}b
	TRAP="${TRAP} /sbin/ifconfig ${myepair}a destroy;"
	trap "${TRAP}" HUP INT ABRT BUS TERM EXIT
fi	# vnet -eq 1

#test for zfs mounted & mount if not
case $zfsfeat in
	1)
		. $zfstool
		zfsmnt ${data}
		[ $? -eq 2 ] && /sbin/zfs mount "${ZPOOL}"
	;;
esac

if [ $baserw -eq 1 ]; then
	path=$data
	[ ! -f "${path}/bin/sh" ] && switch_baserw ${path} 1
	[ -f ${mount_fstab} ] && /bin/rm -f ${mount_fstab}
fi

if [ $baserw -eq 0 -a ! -f "${mount_fstab}" ]; then
	switch_baserw ${path} 2
fi

# MD area
if [ -n "${mdsize}" -a "${mdsize}" != "0" ]; then
	cbsd mountmd mdfile="${MDFILE}" jroot="${data}"
fi

if [ "${ver}" != "empty" ]; then
	[  -d "${data}/etc" -o -d "${data}/bin" ] || err 1 "${MAGENTA}No such data structure: ${GREEN}${data}${NORMAL}"
fi

# cp local default resolv.conf skel
if [ ${floatresolv} -eq 1 -a "${ver}" != "empty" ]; then
	makeresolv jname=${jname}
fi

if [ $applytpl -eq 1 -a "${ver}" != "empty" ]; then
	${SYSRC_CMD} -qf ${data}/etc/rc.conf hostname="${host_hostname}" >/dev/null
fi

MOUNTOPT=""

if [ "$mount_src" = "1" ]; then
	SRCDIR="${srcdir}/src_${ver}/src"
	MOUNTOPT="${MOUNTOPT} -s ${SRCDIR}"
fi

if [ "$mount_obj" = "1" ]; then
	SRCDIR="${srcdir}/obj_${arch}_${ver}/obj"
	MOUNTOPT="${MOUNTOPT} -o ${SRCDIR}"
fi

if [ "$mount_kernel" = "1" ]; then
	SRCDIR="${basejaildir}/kernel_GENERIC_${arch}_${ver}/boot/kernel"
	MOUNTOPT="${MOUNTOPT} -k ${SRCDIR}"
fi

if [ "$mount_ports" = "1" ]; then
	MOUNTOPT="${MOUNTOPT} -p /usr/ports -d ${data}/var/cache/distfiles"
fi

[ "${ver}" != "empty" ] && mountbase -v ${ver} -a ${arch} ${MOUNTOPT}

mount_jail_fstab

exec_master_prestart
external_exec_master_script "master_prestart.d"
exec_prestart

if [ "$ver" = "empty" ]; then
	path="/"
	exec_start="${jailsysdir}/${jname}/run.sh"
else
	[ ! -d "${data}/var/cache/pkg" ] && mkdir -p "${data}/var/cache/pkg"
fi

#determine that jail is FreeBSD. Useful for vnet operation in makejconf and
is_freebsd=0

if [ $baserw -eq 1 ]; then
	elftest=${data}/bin/sh
else
	elftest="${BASE_DIR}/bin/sh"
fi
[ -f "${elftest}" ] && osname=$( ${miscdir}/elf_tables --osname ${elftest} )

[ "${osname}" = "freebsd" ] && is_freebsd=1

# check for freshed version of the base via elf from /bin/sh
if [ ${is_freebsd} -eq 1 -a ${baserw} -eq 1 ]; then
	if [ -n ${BASE_DIR} -a -f "${BASE_DIR}/bin/sh" ]; then
		baseelf=$( ${miscdir}/elf_tables --ver ${BASE_DIR}/bin/sh 2>/dev/null )
		jailelf=$( ${miscdir}/elf_tables --ver ${elftest} 2>/dev/null )
		if [ ${baseelf} -gt ${jailelf} ]; then
			${ECHO} "${BOLD}Notice: ${MAGENTA}You have a more recent version of the base in ${BASE_DIR} (${GREEN}${baseelf}${MAGENTA}/${GREEN}${jailelf}${MAGENTA}).${NORMAL}"
			${ECHO} "${BOLD}Notice: ${MAGENTA}Please consider upgrading jail base via ${GREEN}cbsd jupgrade${NORMAL}"
		fi
	fi
fi

set -e
makejconf jname=${jname} out=${ftmpdir}/${jname}.conf ip6wa=${HAVE_IPV6} epair=${myepair}b fbsd=${is_freebsd}
set +e

#rctl/limits area
jrctl jname=$jname mode=set
. ${workdir}/rctl.subr
[ -z "${nice}" ] && nice="0"
[ "${nice}" != "0" ] && ${ECHO} "${MAGENTA}jail renice: ${GREEN}${nice}${NORMAL}"
#

unset_changes_20140930
set_changes_20140930

${ECHO} "${MAGENTA}Starting jail: ${GREEN}${jname}, parallel timeout=${parallel}${NORMAL}"
TRAP="${TRAP} /bin/rm -f ${ftmpdir}/jstart.${jname}.$$;"
trap "${TRAP}" HUP INT ABRT BUS TERM EXIT

# check that the emulator is able to execute commands
res=$( emulator_exec_check )

if [ -n "${res}" ]; then
	$ECHO "${MAGENTA}Emulator error: ${GREEN}${res}${NORMAL}"
	jcleanup jname="${jname}"
	exit 0
fi

/usr/sbin/daemon -p ${ftmpdir}/jstart.${jname}.$$ /usr/bin/nice -n ${nice} /usr/sbin/jail -f ${ftmpdir}/${jname}.conf -c ${jname}

trap "" HUP INT ABRT BUS TERM EXIT
[ -f "${ftmpdir}/jstart.${jname}.$$" ] && cbsd_pwait --pid=$( /bin/cat ${ftmpdir}/jstart.${jname}.$$ ) --timeout=${parallel}

# make id file
UNDHOSTNAME=$( echo ${hostname} |/usr/bin/tr  "." "_" )
FID="/var/run/jail_${UNDHOSTNAME}.id"
echo ${ST} > ${FID}
sync
exec_cbsdjail_first_boot

external_exec_script "start.d"
exec_master_poststart
external_exec_master_script "master_poststart.d"
exec_poststart

get_jid
if [ $myjid -gt 0 ]; then
	status="1"
	# make shared lock
	echo "${nodename}" > ${jaillock}
	chown ${cbsduser}:${cbsduser} ${jaillock}
else
	status="0"
fi
cbsdsql local UPDATE jails SET jid=$myjid,status=$status where jname=\"${jname}\"
/bin/rm -f ${ftmpdir}/${jname}.conf
fwcounter
expose jname=${jname} mode=apply
[ ${myjid} -gt 0 -a ${cpuset} != "0" ] && /usr/bin/cpuset -c -l ${cpuset} -j ${myjid}
exit 0