svdup

#!/bin/sh

# Default DNS for name resolve
DEFAULT_DNS="1.1.1.1"
DEFAULT_DNS_BACKUP="1.0.0.1"


# Default base software versions:
PICKED_BUILD_VERSION="${1:-current-stable}"
GIT_VERSION="2.26.2"
ZSH_VERSION="5.8.1"


# Literals:
ZPOOL="${ZPOOL:-zroot}"
SOFTWARE_DIR="/Software"
SERVICES_DIR="/Services"
SHARED_DIR="/Shared"
CACHE_DIR="${SHARED_DIR}/Cache"
SYSTEM_IGNITERS_DIR="${SHARED_DIR}/Igniters"
SYSTEM_DNS_DIR="${SHARED_DIR}/DNS"
SYSTEM_NETIDS_DIR="${SHARED_DIR}/NetIDs"
SYSTEM_IPS_DIR="${SHARED_DIR}/IPs"
SERVED_BINARY_RESOURCE="https://software.verknowsys.com/binary/ServeD"
PATH="/Software/Git/exports:/Software/Zsh/exports:/bin:/usr/bin:/sbin:/usr/sbin"
HOME_DIR="/User"
SERVED_REPOSITORY="https://github.com/VerKnowSys/svdOS.git"
SOFIN_REPOSITORY="https://github.com/VerKnowSys/sofin.git"
SOFIN_BIN_REPO="https://software.verknowsys.com/binary"
# BEADM_REPOSITORY="https://github.com/vermaden/beadm.git"
ZSH_AUTOSUGGESTIONS_REPO="https://github.com/VerKnowSys/zsh-autosuggestions"
ORIGINS_RESOURCE="https://raw.githubusercontent.com/VerKnowSys/sofin/master/share/sofin/origins"
LZ4_BIN_RESOURCE="https://raw.githubusercontent.com/VerKnowSys/svdOS/master/usr/bin/lz4"
ZFSX_EXT=".zfsx"
ZFSP_EXT=".zfsp"
ORIGIN="origin"
SYSTEM_NAME="FreeBSD"
PROD_OS_ARCH="amd64"
DATE_NOW="$(date +%F-%H%M 2>/dev/null)"
LOG_FILE="/var/log/svdUP-${DATE_NOW:-common}.log"

# use last stable version
PROD_INITIAL_OS="13.0"
PROD_OS_BASE="${SYSTEM_NAME}-${PROD_INITIAL_OS}-${PROD_OS_ARCH}"


get_the_current () {
    # make sure all ROOT datasets are not read only before proceeding:
    for _mnt in $(zfs mount 2>/dev/null | grep ROOT 2>/dev/null | cut -f1 -d' ' 2>/dev/null); do
        zfs set readonly=off "${_mnt}" >> "${LOG_FILE}" 2>> "${LOG_FILE}"
    done
    zfs list "${ZPOOL}${SYSTEM_IGNITERS_DIR}" >/dev/null 2>&1 || \
    {
        zfs create -o \
            mountpoint=none \
            "${ZPOOL}${SHARED_DIR}";
        zfs create -o \
            mountpoint="${SYSTEM_IGNITERS_DIR}" \
            "${ZPOOL}${SYSTEM_IGNITERS_DIR}";
        zfs create -o \
            mountpoint="${SYSTEM_DNS_DIR}" \
            "${ZPOOL}${SYSTEM_DNS_DIR}";
        zfs create -o \
            mountpoint="${SYSTEM_NETIDS_DIR}" \
            "${ZPOOL}${SYSTEM_NETIDS_DIR}";
        zfs create -o \
            mountpoint="${SYSTEM_IPS_DIR}" \
            "${ZPOOL}${SYSTEM_IPS_DIR}";
        zfs create -o compression=lz4 -o checksum=fletcher4 \
            -o primarycache=metadata -o secondarycache=metadata \
            -o mountpoint="${CACHE_DIR}" \
            "${ZPOOL}${CACHE_DIR}";
    } \
        >> "${LOG_FILE}" 2>> "${LOG_FILE}"

    mkdir -p "${DESTBASE}" >/dev/null 2>&1
    fetch "${SERVED_BINARY_RESOURCE}/${PICKED_BUILD_VERSION}" -o "${CACHE_DIR}/current-version-snapshot" >> "${LOG_FILE}" 2>> "${LOG_FILE}" && \
        cat "${CACHE_DIR}/current-version-snapshot" 2>/dev/null && \
            return 0
    return 1
}


write_default_resolv_conf () {
    printf "%b\n%\n" \
        "nameserver ${DEFAULT_DNS}" \
        "nameserver ${DEFAULT_DNS_BACKUP}" > /etc/resolv.conf
}


fail_with_no_network () {
    echo "Failed to connect with outside network!"
    exit 1
}


fetch_and_install_lz4 () {
    fetch --no-verify-peer "${LZ4_BIN_RESOURCE}" -o /usr/bin/lz4
    chmod 755 /usr/bin/lz4
    ln -fs /usr/bin/lz4 /usr/bin/lz4cat
}


prepare_and_fetch_os_snapshot () {
    ping -c1 -t3 task.pl >> "${LOG_FILE}" 2>> "${LOG_FILE}" || \
        write_default_resolv_conf
    ping -c1 -t3 task.pl >> "${LOG_FILE}" 2>> "${LOG_FILE}" || \
        fail_with_no_network
    test -f /etc/ssl/cert.pem || \
        fetch --no-verify-peer https://curl.haxx.se/ca/cacert.pem -o /etc/ssl/cert.pem || \
        fetch --no-verify-peer http://software.verknowsys.com/source/cacert.pem -o /etc/ssl/cert.pem || \
            fail_with_no_network
    test -x /usr/bin/lz4 || \
        fetch_and_install_lz4

    test -f "${CACHE_DIR}/${MOST_RECENT}" || \
        fetch "${SERVED_BINARY_RESOURCE}/${MOST_RECENT}" -o "${CACHE_DIR}/${MOST_RECENT}" \
            >> "${LOG_FILE}" && return 0
    return 1
}


determine_boot_disk_device () {
    # Regular disk - Native/ Vmware virtualisation defaults:
    for _dsk in $(gpart list -a 2>/dev/null | egrep -i "Geom name:" 2>/dev/null | sed 's/^.*\: //' 2>/dev/null); do
        zpool status "${ZPOOL}" 2>/dev/null | egrep -i "${_dsk}p[0-9]+" >/dev/null 2>&1 && \
            printf '%s\n' "${_dsk}" && \
            return 0

        # gpt/diskN disk, match by label - DO hypervisor defaults:
        for _dsk_label in $(gpart list "${_dsk}" 2>/dev/null | egrep -i "label:" 2>/dev/null | egrep -i "disk" | sed 's/^.*\: //' 2>/dev/null); do
            zpool status "${ZPOOL}" 2>/dev/null | egrep -i "gpt/${_dsk_label}" >/dev/null 2>&1 && \
                printf '%s\n' "${_dsk}" && \
                return 0
        done
    done

    return 1
}


install_boot_environment () {
    zfs list "${ZPOOL}/ROOT/${NEW_BOOT_ENV}" >/dev/null 2>&1
    if [ "0" = "${?}" ]; then
        echo "svdUP: Boot environment: ${NEW_BOOT_ENV} is already received! Redundant operation cancelled!"
        return 6
    else
        echo "svdUP: Receiving new boot env: ${ZPOOL}/ROOT/${NEW_BOOT_ENV}"
        lz4cat "${CACHE_DIR}/${MOST_RECENT}" 2>/dev/null | zfs receive -u "${ZPOOL}/ROOT/${NEW_BOOT_ENV}" && \
            echo "svdUP: Snapshot file: ${MOST_RECENT%%.zfsx} was received as new boot environment: ${ZPOOL}/ROOT/${NEW_BOOT_ENV}!" && \
            return 0
        return 1
    fi
}


set_std_zfs_properties () {
    zfs set sync=standard "${ZPOOL}/ROOT/${NEW_BOOT_ENV}"
    zfs set checksum=fletcher4 "${ZPOOL}/ROOT/${NEW_BOOT_ENV}"
    zfs set readonly=off "${ZPOOL}/ROOT/${NEW_BOOT_ENV}"
    echo "svdUP: Production params were set for ROOT dataset: ${ZPOOL}/ROOT/${NEW_BOOT_ENV}"
    return 0
}


mount_boot_env () {
    bectl mount "${NEW_BOOT_ENV}" "${DESTBASE}" >/dev/null && \
        echo "svdUP: Mounted new environment: ${NEW_BOOT_ENV} with base system: ${DESTBASE##*/}" \
            && return 0
    return 1
}


configure_and_commit_boot_env () {
    _boot_device="${1}"

    mount_boot_env && \
        test -f "${DESTBASE}/boot/pmbr" || exit 177

    mkdir -p \
        "${DESTBASE}/var/www" \
        "${DESTBASE}${CACHE_DIR}" \
        "${DESTBASE}${SYSTEM_IGNITERS_DIR}" \
        "${DESTBASE}${SYSTEM_DNS_DIR}" \
        "${DESTBASE}${SYSTEM_NETIDS_DIR}" \
        "${DESTBASE}${SYSTEM_IPS_DIR}"

    echo "svdUP: Synchronizing system configuration + svdOS specific utils and libs…"
    cp -xpfR "/etc" "${DESTBASE}"
    install -v "/boot/loader.conf" "${DESTBASE}/boot/"
    install -v "/boot/loader.conf.local" "${DESTBASE}/boot/"

    echo "svdUP: Synchronizing system crontabs…"
    mkdir -p "${DESTBASE}/var/cron"
    cp -xpfR "/var/cron/tabs" "${DESTBASE}/var/cron"

    echo "svdUP: Setting setuid for chown"
    chmod +s "${DESTBASE}/usr/sbin/chown"

    echo "svdUP: Synchronizing svdOS utilities…"
    if [ -x "/usr/bin/gvr" ]; then
        echo "svdUP: Syncing: Jail governor (gvr)"
        install -v "/usr/bin/gvr" "${DESTBASE}/usr/bin/"
    fi
    if [ -x "/usr/bin/igni" ]; then
        echo "svdUP: Syncing: Service igniter (igni)"
        install -v "/usr/bin/igni" "${DESTBASE}/usr/bin/"
    fi
    if [ -f "/usr/lib/libkvmpro.a" ]; then
        echo "svdUP: Syncing: kvmpro static library (libkvmpro.a)"
        install -v "/usr/lib/libkvmpro.a" "${DESTBASE}/usr/lib/"
    fi
    if [ -f "/usr/lib/libkvmpro.so" ]; then
        echo "svdUP: Syncing: kvmpro dynamic library (libkvmpro.so)"
        install -v "/usr/lib/libkvmpro.so" "${DESTBASE}/usr/lib/"
    fi
    if [ -x "/usr/bin/sysapi" ]; then
        echo "svdUP: Syncing: SysAPI daemon (sysapi)"
        install -v "/usr/bin/sysapi" "${DESTBASE}/usr/bin/"
    fi
    if [ -x "/usr/bin/lz4" ]; then
        echo "svdUP: Syncing: Lz4 binary utility"
        install -v "/usr/bin/lz4" "${DESTBASE}/usr/bin/lz4"
        ln -fs "/usr/bin/lz4" "${DESTBASE}/usr/bin/lz4cat"
    fi
    if [ -x "/usr/bin/rsync" ]; then
        echo "svdUP: Syncing: Rsync"
        install -v "/usr/bin/rsync" "${DESTBASE}/usr/bin/rsync"
    elif [ -x "/Software/Rsync/bin/rsync" ]; then
        echo "svdUP: Syncing: Rsync (Sofin version)"
        install -v "/Software/Rsync/bin/rsync" "${DESTBASE}/usr/bin/rsync"
    fi

    echo "svdUP: Cleaning: build-host specific mounts"
    sed -i '' -e '/src/ d' "${DESTBASE}/etc/fstab"
    sed -i '' -e '/obj/ d' "${DESTBASE}/etc/fstab"

    gpart bootcode \
        -b "${DESTBASE}/boot/pmbr" \
        -p "${DESTBASE}/boot/gptzfsboot" \
        -i 1 \
        "${_boot_device}" >/dev/null && \
        echo "svdUP: Gpart: Bootcode installed for environment: ${NEW_BOOT_ENV}. Boot device is: ${_boot_device}"

    bectl umount -f "${NEW_BOOT_ENV}" >/dev/null && \
        echo "svdUP: Unmounted new environment: ${NEW_BOOT_ENV}" && \
            return 0

    return 1
}


activate_new_boot_env () {
    zfs rename \
        "${ZPOOL}/ROOT/${NEW_BOOT_ENV}@--head--" \
        "@svdup-${DATE_NOW}" 2>/dev/null

    bectl activate "${NEW_BOOT_ENV}" >/dev/null && \
        echo "svdUP: Commited and activated new boot environment: ${NEW_BOOT_ENV}" && \
        return 0
    return 1
}


configure_base_zfs_datasets () {
    echo "svdUP: Configuring datasets, installing base software, setting HOME to ${HOME_DIR}, Zsh as default shell"
    for _dataset_origin in User Services Software; do
        zfs list "${ZPOOL}/${_dataset_origin}" >/dev/null 2>&1
        if [ "${?}" != "0" ]; then
            zfs create \
                -o mountpoint=none \
                -o casesensitivity=sensitive \
                -o readonly=off \
                -o utf8only=on \
                -o dedup=off \
                -o checksum=fletcher4 \
                -o atime=off \
                -o aclmode=discard \
                -o exec=on \
                -o compression=lz4 \
                "${ZPOOL}/${_dataset_origin}" && \
                echo "svdUP: Created origin base: ${ZPOOL}/${_dataset_origin}"
        fi

        zfs list "${ZPOOL}/${_dataset_origin}/root" >/dev/null 2>&1
        if [ "${?}" != "0" ]; then
            # fetch origin!
            fetch -o \
                "${CACHE_DIR}/${_dataset_origin}-${ORIGIN}.zfsx" \
                "${ORIGINS_RESOURCE}/${_dataset_origin}-${ORIGIN}.zfsx"
            lz4cat "${CACHE_DIR}/${_dataset_origin}-${ORIGIN}.zfsx" | zfs receive -v "${ZPOOL}/${_dataset_origin}/root@${ORIGIN}"
            zfs set mountpoint="/${_dataset_origin}" "${ZPOOL}/${_dataset_origin}/root"
            zfs set sharenfs=off "${ZPOOL}/${_dataset_origin}/root"

            case "${_dataset_origin}" in
                Services)
                    fetch -o "${CACHE_DIR}/Zsh-${ZSH_VERSION}${ZFSP_EXT}" "${SOFIN_BIN_REPO}/Common/Zsh-${ZSH_VERSION}-${SYSTEM_NAME}${ZFSP_EXT}" && lz4cat "${CACHE_DIR}/Zsh-${ZSH_VERSION}${ZFSP_EXT}" | zfs receive -v "${ZPOOL}${SERVICES_DIR}/root/Zsh"
                    fetch -o "${CACHE_DIR}/Git-${GIT_VERSION}${ZFSP_EXT}" "${SOFIN_BIN_REPO}/Common/Git-${GIT_VERSION}-${SYSTEM_NAME}${ZFSP_EXT}" && lz4cat "${CACHE_DIR}/Git-${GIT_VERSION}${ZFSP_EXT}" | zfs receive -v "${ZPOOL}${SERVICES_DIR}/root/Git"
                    ;;

                Software)
                    fetch -o "${CACHE_DIR}/Zsh-${ZSH_VERSION}${ZFSX_EXT}" "${SOFIN_BIN_REPO}/${PROD_OS_BASE}/Zsh-${ZSH_VERSION}-${PROD_OS_BASE}${ZFSX_EXT}"
                    zfs destroy -r "${ZPOOL}${SOFTWARE_DIR}/root/Zsh" 2>/dev/null
                    lz4cat "${CACHE_DIR}/Zsh-${ZSH_VERSION}${ZFSX_EXT}" | zfs receive -v "${ZPOOL}${SOFTWARE_DIR}/root/Zsh"

                    fetch -o "${CACHE_DIR}/Git-${GIT_VERSION}${ZFSX_EXT}" "${SOFIN_BIN_REPO}/${PROD_OS_BASE}/Git-${GIT_VERSION}-${PROD_OS_BASE}${ZFSX_EXT}"
                    zfs destroy -r "${ZPOOL}${SOFTWARE_DIR}/root/Git" 2>/dev/null
                    lz4cat "${CACHE_DIR}/Git-${GIT_VERSION}${ZFSX_EXT}" | zfs receive -v "${ZPOOL}${SOFTWARE_DIR}/root/Git"

                    egrep "/Software/Zsh/exports/zsh" /etc/shells || \
                        echo "/Software/Zsh/exports/zsh" >> /etc/shells
                    chsh -s /Software/Zsh/exports/zsh root
                    ;;

                User)
                    _new_home_mounted="$(zfs get -H -o value mounted "${ZPOOL}/${_new_home_mounted}/root" 2>/dev/null)"
                    if [ "yes" != "${_new_home_mounted}" ]; then
                        zfs mount "${ZPOOL}/${_dataset_origin}/root"
                        test -d "/root/.ssh" && cp -rf "/root/.ssh" "/${_dataset_origin}/"
                    fi
                    pw user mod root -d "/${_dataset_origin}"
                    pw user mod toor -d "/${_dataset_origin}"
                    ;;
            esac
        fi
    done
}


upgrade_projects_sync_configuration () {
    zfs list "${ZPOOL}/Projects" >/dev/null 2>&1
    if [ "${?}" != "0" ]; then
        zfs create \
            -o mountpoint=/Projects \
            -o casesensitivity=sensitive \
            -o readonly=off \
            -o utf8only=on \
            -o dedup=off \
            -o checksum=fletcher4 \
            -o atime=off \
            -o exec=on \
            -o compression=lz4 \
            "${ZPOOL}/Projects" && \
            echo "svdUP: Created origin base: ${ZPOOL}/Projects"
    fi

    test -f "${SERVICES_DIR}/Git/etc/ssl/cert.pem" || cp -f /etc/ssl/cert.pem "${SERVICES_DIR}/Git/etc/ssl/cert.pem"
    _timestamp_now="$(date +%F-%H%M-%s 2>/dev/null)"

    cd /Projects
    ${SOFTWARE_DIR}/Git/bin/git clone "${SOFIN_REPOSITORY}" >/dev/null
    cd sofin
    ${SOFTWARE_DIR}/Git/bin/git reset --hard >/dev/null 2>&1
    # Install Sofin - software installer:
    bin/upgrade

    # cd /Projects
    # ${SOFTWARE_DIR}/Git/bin/git clone "${BEADM_REPOSITORY}" >/dev/null
    # cd beadm
    # ${SOFTWARE_DIR}/Git/bin/git reset --hard >/dev/null 2>&1
    # ${SOFTWARE_DIR}/Git/bin/git pull origin master
    # rm -f "beadm.1.gz"
    # gzip "beadm.1" >/dev/null
    # install -v "beadm.1.gz" "/usr/share/man/man1/"
    # install -v "beadm" "/usr/sbin/beadm"

    cd /Projects
    ${SOFTWARE_DIR}/Git/bin/git clone "${SERVED_REPOSITORY}"
    cd svdOS
    ${SOFTWARE_DIR}/Git/bin/git reset --hard >/dev/null 2>&1
    ${SOFTWARE_DIR}/Git/bin/git pull origin master

    # always install default sysctl.conf.served:
    install -v "etc/sysctl.conf.served" "/etc/sysctl.conf.served"

    # System config: Install rc.conf and rc.conf.local, only if doesn't exist:
    test -f "/etc/rc.local" || install -v "etc/rc.local" "/etc/rc.local"
    test -f "/etc/rc.conf.local" || install -v "etc/rc.production.conf" "/etc/rc.conf.local"

    # Installing Zsh configuration:
    cp -fr "shell/" "/etc/zsh"
    ln -fs "/etc/zsh/zshenv" "/etc/zshenv"
    ln -fs "/etc/zsh/zshrc" "/etc/zshrc"

    cd /
}


move_db_to_separate_writable_dataset () {
    mv "/var/db" "/tmp/db"
    zfs create -o mountpoint="/var/db" "${ZPOOL}/var/db" 2>/dev/null
    zfs set readonly=off "${ZPOOL}/var/db"
    zfs mount "${ZPOOL}/var/db" 2>/dev/null
    cp -fR /tmp/db/* "/var/db/"
}

postinstall_cleanup () {
    echo "svdUP: Performing post-installation cleanup"
    zfs list "${ZPOOL}/var/db" >/dev/null 2>&1 || move_db_to_separate_writable_dataset
    zfs list "${ZPOOL}/var/mail" >/dev/null 2>&1 && zfs destroy -r "${ZPOOL}/var/mail"
    zfs list "${ZPOOL}/usr/ports" >/dev/null 2>&1 && zfs destroy -r "${ZPOOL}/usr/ports"
    zfs set primarycache=all "${ZPOOL}/ROOT"
    zfs set secondarycache=all "${ZPOOL}/ROOT"
    zfs set checksum=fletcher4 "${ZPOOL}/ROOT"
    zfs set sync=standard "${ZPOOL}/ROOT"
    zfs set readonly=off "${ZPOOL}/ROOT"
    zfs set readonly=off "${ZPOOL}/ROOT/${NEW_BOOT_ENV}"
    zfs set readonly=off "${ZPOOL}/var"

    rm -rf "/.profile" "/.cshrc" "/proc"

    mkdir -p "${HOME_DIR}/.ssh"
    grep 'verknowsys.com' "${HOME_DIR}/.ssh/config" >/dev/null 2>&1 \
        || printf 'Host *.verknowsys.com verknowsys.com\nPort 60022\n\n' >> "${HOME_DIR}/.ssh/config"

    # only on hardened systems
    if [ -x "/usr/bin/hbsdcontrol" ]; then
        grep -E 'rlimit memlock 0' "/etc/ntp.conf" >/dev/null 2>&1 \
            || printf "\n%s" "rlimit memlock 0" >> "/etc/ntp.conf"
        grep -E 'tinker panic 0' "/etc/ntp.conf" >/dev/null 2>&1 \
            || printf "\n%s" "tinker panic 0" >> "/etc/ntp.conf"
    fi
}



#
# main():
#

case "$(uname)" in
    "${SYSTEM_NAME}")
        ;;
    *)
        echo "Currently only ${SYSTEM_NAME} x86_64 hosts are supported by svdup!"
        exit 1
esac

# Most recent ServeD-OS snapshot:
MOST_RECENT="$(get_the_current)"
if [ -z "${MOST_RECENT}" ]; then
    echo "svdUP: svdUP: Fatal: Error: MOST_RECENT value is empty!"
    exit 5
fi

NEW_BOOT_ENV="${MOST_RECENT%%.zfsx}"
DESTBASE="${SHARED_DIR}/${NEW_BOOT_ENV}"

export PATH MOST_RECENT ZPOOL DESTBASE NEW_BOOT_ENV

# main():
BOOT_DEVICE="$(determine_boot_disk_device)"
if [ -z "${BOOT_DEVICE}" ]; then
    echo "svdUP: Fatal: Error: main(): Got empty boot device!"
    exit 6
fi
echo "svdUP: Default boot device: ${BOOT_DEVICE}"
echo "svdUP: Tasks in progress…" && \
    prepare_and_fetch_os_snapshot && \
    configure_base_zfs_datasets && \
    upgrade_projects_sync_configuration && \
    install_boot_environment && \
    set_std_zfs_properties && \
    configure_and_commit_boot_env "${BOOT_DEVICE}" && \
    postinstall_cleanup && \
    activate_new_boot_env && \
    echo "svdUP: All tasks were completed successfully!" && \
    exit 0

exit 1