[WiP] ECS 3.1 Full GA and CE support (#367)

* Use nodeId instead of the node IP to create data store

* Fix errors getting node ID

* fix 3.1 patch again

* migrate cm.object.properties/'MustHaveEnoughResources=false' into Dockerfile

* Run cf_client in container for new low partition count vars

* Run cf_client in container for new low partition count vars

* migrate cf_client variable settings into Dockerfile

* update comments in Dockerfile for 3.1.0.0

patches/3.1.0.0/Dockerfile

@@ -1,19 +1,23 @@
 # Fixes to the default 3.1.0.0 reduced image.
-# RC4
+
+# Build on RC4 object image (GA release)
 FROM emcvipr/object:3.1.0.0-95266.ab2753a-reduced
 
 # Increase memory for transformsvc
 RUN sed -i s/-Xmx128m/-Xmx512m/ /opt/storageos/bin/transformsvc
 
 # Fix disk partitioning script
 RUN sed -i '/VMware/ s/$/ \&\& [ ! -e \/data\/is_community_edition ]/' /opt/storageos/bin/storageserver-partition-config.sh
-
 RUN /usr/bin/chmod +x /opt/storageos/bin/storageserver-partition-config.sh
 
-# Make vnest use separate thread pools to prevent deadlock
-RUN printf "\n# Use separate thread pools to prevent deadlock in vnest init\nobject.UseSeparateThreadPools=true\n" >> /opt/storageos/conf/vnest.object.properties
-
+# Set VNets useSeperateThreadPools to True
 RUN f=/opt/storageos/conf/vnest-common-conf-template.xml; grep -q "object.UseSeparateThreadPools" $f || sed -i '/properties id="serviceProperties"/a \ \ \ \ \ \ \ \ <prop key="object.UseSeparateThreadPools">true</prop>' $f
 
 # Set georeceiver's initialBufferNumOnHeap to something smaller for CE
 RUN f=/opt/storageos/conf/georeceiver-conf.xml; grep -q 'name="initialBufferNumOnHeap" value="10"' $f ||  sed -i 's/name="initialBufferNumOnHeap" value="60"/name="initialBufferNumOnHeap" value="10"/' $f
+
+# Configure CM Object properties: Disable minimum storage device count
+RUN f=/opt/storageos/conf/cm.object.properties; grep -q 'MustHaveEnoughResources=false' $f ||  sed -i 's/MustHaveEnoughResources=true/MustHaveEnoughResources=false/' $f
+
+# Allow allocation of different blocks of a chunk to be stored on the same partition
+RUN sed -i 's#<config:boolean name="allowAllocationOnIgnoredPartitions" value="false" description="If set to true, different blocks in one chunk may be allocated on the same partition"/>#<config:boolean name="allowAllocationOnIgnoredPartitions" value="true" description="If set to true, different blocks in one chunk may be allocated on the same partition"/>#g' /opt/storageos/conf/ssm-cf-conf.xml

ui/ansible/clicmd_start.yml

@@ -8,3 +8,4 @@
   hosts: data_node
   roles:
     - common_start_enable_service
+#    - common_run_cf_client

ui/ansible/roles/common_run_cf_client/tasks/main.yml

@@ -0,0 +1,8 @@
+- name: Common | Run cf_client in ecs-storageos container
+  command: /usr/bin/docker exec ecs-storageos /opt/storageos/tools/cf_client --user emcservice --password ChangeMe --set --name com.emc.ecs.ssm.allocation.allowAllocationOnIgnoredPartitions --value true --reason allow
+
+#The reason is L1 DT table can’t allocate chunk from L2 tables and keep read only, then it block the system finish initialize.
+#The reason can’t allocate chunk is because we only has one partition (it may cause by MustHaveEnoughResources=true during installing, or there will 5 data partitions in a vm node). We need enable another flag in ssm to allow allocate all blocks in one partition.
+#luna:/opt/storageos/tools # ./cf_client --user emcservice --password ChangeMe --set --name com.emc.ecs.ssm.allocation.allowAllocationOnIgnoredPartitions --value true --reason allow
+#After change the value, I add VDC and RG manually from GUI. ECS works now.
+

ui/ecsconfig.py

@@ -567,15 +567,21 @@ def sp_add_node(sp_id, node_ip):
 
         node_dict = conf.ecs.get_node_options(node_ip)
 
+        # Obtain the nodeId from the node IP address
+        # In ECS < 3.1, nodeId is the IP address
+        # In ECS >= 3.1, nodeId is an autogenerated UUID
+        nodes = conf.api_client.node.list()
+        node_info = list(filter(lambda x: x['ip'] == node_ip, nodes['node']))
+
         kwargs = {"name": node_ip,
                   "description": node_dict['description'],
-                  "node_id": node_ip,
+                  "node_id": node_info[0]['nodeid'],
                   "storage_pool_id": sp_id}
         """
         def create(self, name, description, node_id, storage_pool_id):
         :param name: User provided name (not verified or unique)
         :param description: User provided description (not verified or unique)
-        :param node_id: IP address for the commodity node
+        :param node_id: ID of the commodity node
         :param storage_pool_id: Desired storage pool ID for creating data store
         :returns a task object
         """

ui/resources/docker/ecs-install-requirements.txt

@@ -11,7 +11,7 @@ ptpython>=0.41,<1.0
 cryptography>=1.9
 pycrypto>=2.6.1,<3.0
 pyOpenSSL>=17.3.0,<18.0
-python-ecsclient>=1.1.5,<2.0
+python-ecsclient>=1.1.6,<2.0
 PyYAML>=3.12,<4.0
 requests>=2.18.4,<3.0
 sarge>=0.1.4,<1.0