Update to 3.3.0

3.3 RC patch

Update Centos Release to 7.6 - deprecate *>7.4 - iterate CE ver.

temp fix

increase sr heap size

3.3 RC patch update

increase memory for sr

Bump memory, fix ref, pin docker version

pin docker version

bump install node version

bump install node version

.github/ISSUE_TEMPLATE.md

@@ -16,4 +16,4 @@
 ```
 
 ---
-Notifies: @padthaitofuhot @captntuttle @adrianmo
+Notifies: @captntuttle

.github/PULL_REQUEST_TEMPLATE.md

@@ -11,4 +11,4 @@
 3.
 
 ---
-@padthaitofuhot @captntuttle @adrianmo
+@captntuttle

bootstrap.sh

@@ -566,6 +566,11 @@ p Installing VM guest additions
     ping_sudo
 fi
 
+### Lock packages that we don't want updated
+v "Locking packages we don't want updated"
+p locking docker packages
+versionlock_packages 2>&1 | log
+ping_sudo
 
 ### Update repo databases and all system packages (again)
 ### This will pick up any updates pulled in from alternate repos.

bootstrap_plugins/centos76.plugin.sh

@@ -0,0 +1,212 @@
+#@IgnoreInspection BashAddShebang
+
+# Copyright (c) 2015 EMC Corporation
+# All Rights Reserved
+#
+# This software contains the intellectual property of EMC Corporation
+# or is licensed to EMC Corporation from third parties.  Use of this
+# software and the intellectual property contained therein is expressly
+# limited to the terms and conditions of the License Agreement under which
+# it is provided by or on behalf of EMC.
+
+# OS Support library for CentOS 7.5
+
+os_supported=true
+
+# Docker binary
+docker_binary='/bin/docker'
+
+# packages to clean up during preflight
+# Don't `yum autoremove curl`.  Yum is a dependency and it will throw errors.
+list_preflight_packages="git nfs-client nfs-tools rsync wget ntp docker-1.13.1-75.git8633870.el7.centos vim pigz gdisk aria2 htop iotop iftop multitail dstat jq python-docker-py dkms qemu-guest-agent open-vm-tools open-vm-tools-desktop docker"
+
+# Do any OS-specific tasks that must be done prior to bootstrap
+do_preflight() {
+    rm_repo_pkg "$list_preflight_packages"
+}
+
+# packages to install before others
+list_prefix_packages='wget curl epel-release yum-utils'
+
+# script to run for installing prefix_packages
+in_prefix_packages() {
+    in_repo_pkg "$list_prefix_packages"
+}
+
+# packages to install
+# list_general_packages='yum-utils git python-pip python-docker-py'
+list_general_packages='git ntp docker-1.13.1-75.git8633870.el7.centos vim rsync pigz gdisk aria2 yum-versionlock'
+
+# script to run for installing general_packages
+in_general_packages() {
+    in_repo_pkg "$list_general_packages"
+#    if ! docker version; then
+#        curl -fsSL https://get.docker.com/ | sudo sh
+#    fi
+    sudo systemctl enable docker
+    sudo systemctl start docker
+    sudo usermod -aG docker $(whoami)
+}
+
+# packages to install after others
+list_suffix_packages='htop iotop iftop multitail dstat jq python-docker-py'
+# list_suffix_packages='htop jq pigz gdisk aria2 python-docker-py'
+
+# packages to lock after installation
+list_lock_packages='docker docker-common docker-client'
+
+# script to run for installing suffix_packages
+in_suffix_packages() {
+    in_repo_pkg "$list_suffix_packages"
+
+    # Install Rocker
+    curl -fsSL ${rocker_artifact_url} \
+    | sudo tar -xzC /usr/local/bin && sudo chmod +x /usr/local/bin/rocker
+}
+
+# packages to install if a VM
+list_vm_packages='dkms qemu-guest-agent open-vm-tools open-vm-tools-desktop'
+
+# command to run for installing vm_packages
+in_vm_packages() {
+    in_repo_pkg "$list_vm_packages"
+    # return 0
+}
+
+versionlock_packages() {
+    lock_pkg "$list_lock_packages"
+}
+
+# command to install one or more os package manager package
+in_repo_pkg() {
+    retry_with_timeout 10 300 sudo yum -y install $*
+}
+
+rm_repo_pkg() {
+    retry_with_timeout 10 300 sudo yum -y autoremove $*
+}
+
+# lock packages that we don't want updated
+lock_pkg() {
+    sudo yum versionlock $*
+}
+
+# command to update all packages in the os package manager
+up_repo_pkg_all() {
+    retry_with_timeout 10 300 sudo yum -y update
+}
+
+# command to rebuild the os package manager's database
+up_repo_db() {
+    retry_with_timeout 10 300 sudo yum -y makecache
+}
+
+# command to set os package manager proxy
+set_repo_proxy_conf() {
+    sudo sed -i -e '/^proxy=/d' /etc/yum.conf
+    echo "proxy=${http_proxy}" \
+        | append /etc/yum.conf
+}
+
+# command to set os package manager to keep its cache
+set_repo_keepcache_conf() {
+    sudo sed -i -e '/^keepcache=/d' /etc/yum.conf
+    echo "keepcache=1" \
+        | append /etc/yum.conf
+}
+
+# idempotent config script to fixup repos to properly use proxycaches
+set_repo_cacheable_idempotent() {
+    sudo sed -i -e 's/^#baseurl=/baseurl=/' /etc/yum.repos.d/*
+    sudo sed -i -e 's/^mirrorlist=/#mirrorlist=/' /etc/yum.repos.d/*
+}
+
+set_repo_mirror_idempotent() {
+    # sudo sed -i -e "s#http:///centos#http://${mirror_val}/centos#g" /etc/yum.repos.d/*
+    sudo sed -i -e "s#http://.*/centos#http://${mirror_val}/centos#g" /etc/yum.repos.d/*
+}
+
+# command to set the proxy for the whole OS
+set_os_proxy() {
+    sudo sed -i -e '/_proxy/d' /etc/environment
+    echo -n "http_proxy=${http_proxy}\nhttps_proxy=${http_proxy}\nftp_proxy=${http_proxy}\n" \
+        | append /etc/environment
+    if $mirror_flag; then
+        echo -n "no_proxy=${mirror_val}\n" | append /etc/environment
+    fi
+}
+
+# command to determine if the OS needs restarting after package updates
+get_os_needs_restarting() {
+    if ! [ -z "$(sudo /usr/bin/needs-restarting)" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+# command to reboot the system
+do_reboot() {
+    sudo reboot
+}
+
+# Command to configure docker's proxy under centos flavored systemd
+set_docker_proxy() {
+    local tmpconf="/etc/systemd/system/docker.service.d/http-proxy.conf"
+    if ! [ -d "$(dirname $tmpconf)" ]; then
+        sudo mkdir "$(dirname $tmpconf)"
+    fi
+    log "sed error is OK here if the proxy config file does not yet exist."
+    sudo sed -i -e '/HTTP_PROXY/d' "$tmpconf"
+    echo "Environment=\"HTTP_PROXY=${http_proxy}\" \"NO_PROXY=localhost,127.0.0.1,$(hostname),$(hostname -f)\"" \
+        | append "$tmpconf"
+    sudo systemctl daemon-reload
+    sudo systemctl restart docker
+    sudo systemctl status docker
+}
+
+# command to add mitm cert to docker trust store
+set_docker_reg_cert() {
+    local registry="${1}"
+    local cert="${2}"
+    if ! [ -d "/etc/docker/certs.d/${registry}" ]; then
+        sudo mkdir -p "/etc/docker/certs.d/${registry}"
+        sudo cp "${cert}" "/etc/docker/certs.d/${registry}/ca.crt"
+    else
+        if [ -f "/etc/docker/certs.d/${registry}/ca.crt" ]; then
+            echo "Reusing existing /etc/docker/certs.d/${registry}/ca.crt"
+        else
+            sudo cp "${cert}" "/etc/docker/certs.d/${registry}/ca.crt"
+        fi
+    fi
+    set_mitm_cert "${cert}"
+    sudo systemctl daemon-reload
+    sudo systemctl restart docker
+    sudo systemctl status docker
+}
+
+# command to add mitm cert to local trust store
+set_mitm_cert() {
+    sudo cp "${1}" "/etc/pki/ca-trust/source/anchors/$(basename ${1}).crt"
+    sudo update-ca-trust extract
+}
+
+do_post_install() {
+    # Disable postfix since we don't need an MTA
+    sudo systemctl disable --now postfix
+}
+
+override_dhcp_dns() {
+    nameserver_list="${1}"
+    sudo sed -i -e 's/PEERDNS="yes"/PEERDNS="no"/' /etc/sysconfig/network-scripts/ifcfg-*
+    sudo sed -i -e '/DNS[0-9]=/d' /etc/sysconfig/network-scripts/ifcfg-*
+    sudo sed -i -e '/nameserver/d' /etc/resolv.conf
+    nsnumber=1
+    for nameserver in $(echo ${nameserver_list} | tr ',' ' '); do
+        echo "nameserver ${nameserver}" | append /etc/resolv.conf
+        for script in /etc/sysconfig/network-scripts/ifcfg-*; do
+            echo "DNS${nsnumber}=${nameserver}" | append "${script}"
+        done
+        nsnumber=$((nsnumber++))
+    done
+}

bootstrap_plugins/os-router.plugin.sh

@@ -58,18 +58,21 @@ route_os() {
 
     case "${os}" in
 
-        centos\ linux\ release\ 7.2*)
-            source ${plugins}/centos72.plugin.sh
-            ;;
-        centos\ linux\ release\ 7.3*)
-            source ${plugins}/centos73.plugin.sh
-            ;;
+#        centos\ linux\ release\ 7.2*)
+#            source ${plugins}/centos72.plugin.sh
+#            ;;
+#        centos\ linux\ release\ 7.3*)
+#            source ${plugins}/centos73.plugin.sh
+#            ;;
         centos\ linux\ release\ 7.4*)
             source ${plugins}/centos74.plugin.sh
             ;;
         centos\ linux\ release\ 7.5*)
             source ${plugins}/centos75.plugin.sh
             ;;
+        centos\ linux\ release\ 7.6*)
+            source ${plugins}/centos76.plugin.sh
+            ;;
 #        dockerized\ centos\ linux\ release\ 7.2*)
 #            source ${plugins}/centos72-docker.plugin.sh
 #            ;;

docs/source/conf.py

@@ -54,9 +54,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'3.0'
+version = u'3.2'
 # The full version, including alpha/beta/rc tags.
-release = u'3.0'
+release = u'3.2.2.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

patches/3.3.0.0/Dockerfile

@@ -0,0 +1,27 @@
+# Fixes to the default 3.3.0.0 reduced image.
+
+# Build on object-reduced image (GA release)
+FROM emcvipr/object:3.3.0.0-108986.c0575c7-reduced
+
+# Fix disk partitioning script
+RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh \
+    && /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh
+
+# Set VNest useSeperateThreadPools to True
+#RUN f=/opt/storageos/conf/vnest-common-conf.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ <prop key="object.UseSeparateThreadPools">true</prop>' $f
+
+# Allow allocation of different blocks of a chunk to be stored on the same partition
+#RUN f=/opt/storageos/conf/ssm-cf-conf.xml; grep -q '<config:boolean name="allowAllocationOnIgnoredPartitions" value="true" description="If set to true, different blocks in one chunk may be allocated on the same partition"/>' $f || sed -i 's#<config:boolean name="allowAllocationOnIgnoredPartitions" value="false" description="If set to true, different blocks in one chunk may be allocated on the same partition"/>#<config:boolean name="allowAllocationOnIgnoredPartitions" value="true" description="If set to true, different blocks in one chunk may be allocated on the same partition"/>#g' /opt/storageos/conf/ssm-cf-conf.xml $f
+
+## Increase memory for transformsvc
+#RUN sed -i s/Xmx128m/Xmx512m/ /opt/storageos/bin/transformsvc
+
+## Set memory for objcontrolsvc
+#RUN sed -i s/Xmx96m/Xmx256m/ /opt/storageos/bin/objcontrolsvc
+
+# Set georeceiver's initialBufferNumOnHeap to something smaller for CE
+#RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="5"' $f ||  sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="5"/' $f
+#RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q '<prop key="object.InitialBufferNumOnHeap">10</prop>' $f || sed -i 's#<prop key="object.InitialBufferNumOnHeap">80</prop>#<prop key="object.InitialBufferNumOnHeap">10</prop>#g' $f
+
+# Configure CM Object properties: Disable minimum storage device count
+#RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f ||  sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f

patches/3.3.0.0/build.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+source image.conf
+
+docker build -t "${IMAGE_REPO}:${IMAGE_VERSION}" .

patches/3.3.0.0/image.conf

@@ -0,0 +1,3 @@
+BASE_IMAGE="emcvipr/object:3.3.0.0-109089.4bf8a5a-reduced"
+IMAGE_REPO="emccorp/ecs-software-3.3.0"
+IMAGE_VERSION="3.3.0.0"

patches/libexec/patch_script.sh

@@ -54,6 +54,7 @@ sed -i s/-Xmx128m/-Xmx512m/ /opt/storageos/bin/transformsvc
 sed -i s/-Xmx2048m/-Xmx3072m/ /opt/storageos/bin/blobsvc
 # sed -i s/-Xmx512m/-Xmx256m/ /opt/storageos/bin/metering
 sed -i s/\$\(_get_mem_fraction_mb\ 428.3m\ 0.0081\)/64/ /opt/storageos/bin/ecsportalsvc
+sed -i s/-Xmx384m/-Xmx768m/ /opt/storageos/bin/sr
 # sed -i s/-Xmx128m/-Xmx64m/ /opt/storageos/bin/dtquery
 sed -i s/-Xmx192m/-Xmx128m/ /opt/storageos/bin/eventsvc
 sed -i s/-Xmx256m/-Xmx128m/ /opt/storageos/bin/resourcesvc

release.conf

@@ -11,6 +11,10 @@
 # DO NOT specify an alternate registry here. Use the -r argument to
 # bootstrap.sh for that.
 #
+### Older versions ( 3.3 )
+# release_artifact="emccorp/ecs-software-3.3.0"
+# release_tag="latest"
+#
 ### Older versions ( 3.2 )
 # release_artifact="emccorp/ecs-software-3.2.0"
 # release_tag="3.2.2.0"

ui/ansible/roles/common_deploy/tasks/main.yml

@@ -168,6 +168,14 @@
     insertafter: '<util:properties id="serviceProperties" location="classpath:systemsvc.object.properties" local-override="true">'
     line: '        <prop key="object.UseSeparateThreadPools">true</prop>'
 
+## sr configuration
+- name: 'Common | Increase memory for sr'
+  lineinfile:
+    dest: /host/sr
+    backrefs: yes
+    regexp: '^(.*)Xmx384m(.*)$'
+    line: '\1Xmx768m\2'
+
 ### Then docker cp them into the configuration container
 - name: Common | Merge configurations into configuration data container
   command: docker cp /host/{{item.split('/')[-1]}} ecs-config:{{item}}

ui/ansible/roles/common_deploy/vars/main.yml

@@ -10,5 +10,6 @@ config_files:
   - /opt/storageos/bin/dtquery
   - /opt/storageos/bin/transformsvc
   - /opt/storageos/bin/objcontrolsvc
+  - /opt/storageos/bin/sr
   - /opt/storageos/conf/georeceiver-conf.xml
   - /opt/storageos/conf/vnest-common-conf.xml

ui/etc/config.yml

@@ -13,7 +13,7 @@
 ---
 ui:
   name: ECS Community Edition Install Node
-  version: 2.9.0r
+  version: 2.9.1r
   host_root_dir: /opt/emc/ecs-install
   state_file: /opt/state.yml
   deploy_file: /opt/deploy.yml
@@ -32,7 +32,7 @@ ui:
   ffx_sem: /opt/ffx.sem
 product:
   name: ECS
-  version: 3.2.2.0
+  version: 3.3.0.0
   vendor: Dell EMC
   flavor: Community Edition
   slogan: Free and Frictionless