Fix cluster configuration schema

cli/src/pcluster/config/cluster_config.py

@@ -838,13 +838,9 @@ class BaseComputeResource(Resource):
     def __init__(
         self,
         name: str,
-        disable_simultaneous_multithreading: bool = None,
     ):
         super().__init__()
         self.name = Resource.init_param(name)
-        self.disable_simultaneous_multithreading = Resource.init_param(
-            disable_simultaneous_multithreading, default=False
-        )
 
     def _register_validators(self):
         self._register_validator(NameValidator, name=self.name)
@@ -1260,13 +1256,17 @@ def __init__(
         min_count: int = None,
         spot_price: float = None,
         efa: Efa = None,
+        disable_simultaneous_multithreading: bool = None,
         **kwargs,
     ):
         super().__init__(**kwargs)
         self.instance_type = Resource.init_param(instance_type)
         self.max_count = Resource.init_param(max_count, default=DEFAULT_MAX_COUNT)
         self.min_count = Resource.init_param(min_count, default=DEFAULT_MIN_COUNT)
         self.spot_price = Resource.init_param(spot_price)
+        self.disable_simultaneous_multithreading = Resource.init_param(
+            disable_simultaneous_multithreading, default=False
+        )
         self.__instance_type_info = None
         efa_supported = self.instance_type_info.is_efa_supported()
         self.efa = efa or Efa(enabled=efa_supported, implied=True)

cli/src/pcluster/schemas/cluster_schema.py

@@ -239,7 +239,7 @@ def make_resource(self, data, **kwargs):
 class HeadNodeStorageSchema(BaseSchema):
     """Represent the schema of storage attached to a node."""
 
-    root_volume = fields.Nested(HeadNodeRootVolumeSchema, metadata={"update_policy": UpdatePolicy.UNSUPPORTED})
+    root_volume = fields.Nested(HeadNodeRootVolumeSchema, metadata={"update_policy": UpdatePolicy.SUPPORTED})
     ephemeral_volume = fields.Nested(
         HeadNodeEphemeralVolumeSchema, metadata={"update_policy": UpdatePolicy.UNSUPPORTED}
     )
@@ -705,7 +705,7 @@ def make_resource(self, data, **kwargs):
 class ClusterIamSchema(BaseSchema):
     """Represent the schema of IAM for Cluster."""
 
-    roles = fields.Nested(RolesSchema)
+    roles = fields.Nested(RolesSchema, metadata={"update_policy": UpdatePolicy.SUPPORTED})
     permissions_boundary = fields.Str(
         metadata={"update_policy": UpdatePolicy.SUPPORTED}, validate=validate.Regexp("^arn:.*:policy/")
     )
@@ -905,7 +905,6 @@ class _ComputeResourceSchema(BaseSchema):
     """Represent the schema of the ComputeResource."""
 
     name = fields.Str(required=True, metadata={"update_policy": UpdatePolicy.UNSUPPORTED})
-    disable_simultaneous_multithreading = fields.Bool(metadata={"update_policy": UpdatePolicy.COMPUTE_FLEET_STOP})
 
 
 class SlurmComputeResourceSchema(_ComputeResourceSchema):
@@ -916,6 +915,7 @@ class SlurmComputeResourceSchema(_ComputeResourceSchema):
     min_count = fields.Int(validate=validate.Range(min=0), metadata={"update_policy": UpdatePolicy.COMPUTE_FLEET_STOP})
     spot_price = fields.Float(validate=validate.Range(min=0), metadata={"update_policy": UpdatePolicy.SUPPORTED})
     efa = fields.Nested(EfaSchema, metadata={"update_policy": UpdatePolicy.COMPUTE_FLEET_STOP})
+    disable_simultaneous_multithreading = fields.Bool(metadata={"update_policy": UpdatePolicy.COMPUTE_FLEET_STOP})
 
     @post_load
     def make_resource(self, data, **kwargs):

cli/src/pcluster/schemas/common_schema.py

@@ -136,8 +136,12 @@ def _is_implied(resource, attr, value):
 class TagSchema(BaseSchema):
     """Represent the schema of Tag section."""
 
-    key = fields.Str(validate=validate.Length(max=128), metadata={"update_policy": UpdatePolicy.UNSUPPORTED})
-    value = fields.Str(validate=validate.Length(max=256), metadata={"update_policy": UpdatePolicy.SUPPORTED})
+    key = fields.Str(
+        required=True, validate=validate.Length(max=128), metadata={"update_policy": UpdatePolicy.UNSUPPORTED}
+    )
+    value = fields.Str(
+        required=True, validate=validate.Length(max=256), metadata={"update_policy": UpdatePolicy.SUPPORTED}
+    )
 
     @post_load
     def make_resource(self, data, **kwargs):

cli/tests/pcluster/schemas/test_cluster_schema/test_cluster_schema_awsbatch/awsbatch.full.yaml

@@ -1,43 +1,43 @@
 Image:
   Os: alinux2  # alinux2 | centos7 | ubuntu1804 | ubuntu2004
-  CustomAmi: ami-12345678  # ami-xxx
+  CustomAmi: ami-12345678
 HeadNode:
-  InstanceType: String  # t2.micro
+  InstanceType: t2.micro
   Networking:
-    SubnetId: subnet-12345678  # subnet-xxx
+    SubnetId: subnet-12345678
     ElasticIp: String  # true|false|EIP-id
     AssignPublicIp: true  # true|false
     AdditionalSecurityGroups:
-      - sg-12345678  # sg-xxxx
+      - sg-12345678
       - sg-23456789
     Proxy:
-      HttpProxyAddress: String  # https://proxy-address:port
+      HttpProxyAddress: https://proxy-address:port
   DisableSimultaneousMultithreading: true
   CustomActions:
     OnNodeStart:
       Script: https://test.tgz  # s3:// | https:// | file://*
       Args:
-        - String  # arg1
-        - stirng2
+        - arg1
+        - arg2
     OnNodeConfigured:
       Script: https://test.tgz  # s3:// | https:// | file://*
       Args:
-        - String  # arg1
-        - stirng2
+        - arg1
+        - arg2
   Ssh:
-    KeyName: String  # ec2-key-name
-    AllowedIps: 1.2.3.4/32  # 1.2.3.4/32
+    KeyName: ec2-key-name
+    AllowedIps: 1.2.3.4/32
   LocalStorage:
     RootVolume:
-      Size: 37  # 35
-      Encrypted: true  # true
+      Size: 37
+      Encrypted: true
       DeleteOnTermination: true
     EphemeralVolume:
-      Encrypted: true  # true
-      MountDir: String  # /scratch
+      Encrypted: true
+      MountDir: /scratch
   Dcv:
-    Enabled: true  # true
-    Port: 8443  # 8443
+    Enabled: true
+    Port: 8443
     AllowedIps: 0.0.0.0/0  # 0.0.0.0/0
 Scheduling:
   Scheduler: awsbatch
@@ -54,7 +54,7 @@ Scheduling:
           Enabled: true  # true
           Id: String
         Proxy:
-          HttpProxyAddress: String  # https://proxy-address:port
+          HttpProxyAddress: https://proxy-address:port
       ComputeResources:  # this maps to a Batch compute environment (initially we support only 1)
         - Name: compute_resource
           InstanceTypes:
@@ -65,7 +65,6 @@ Scheduling:
           DesiredvCpus: 10
           MaxvCpus: 20
           SpotBidPercentage: 50
-          DisableSimultaneousMultithreading: false
 SharedStorage:
   - MountDir: /my/mount/point
     StorageType: Ebs
@@ -90,57 +89,53 @@ SharedStorage:
       KmsKeyId: String  # id-xxx
       PerformanceMode: maxIO  # generalPurpose | maxIO
       ThroughputMode: provisioned  # bursting | provisioned
-      ProvisionedThroughput: 1024  # 1024
-      FileSystemId: fs-12345678  # fs-xxxx
+      ProvisionedThroughput: 1024
+      FileSystemId: fs-12345678
   - MountDir: /my/mount/point3
     StorageType: FsxLustre
     Name: name3
     FsxLustreSettings:
       StorageCapacity: 3600  # 3600
       DeploymentType: SCRATCH_1  # PERSISTENT_1 | SCRATCH_1 | SCRATCH_2
       DataCompressionType: LZ4
-      ImportedFileChunkSize: 10  # 1024
-      ExportPath: String  # s3://bucket/folder
-      ImportPath: String  # s3://bucket
-      WeeklyMaintenanceStartTime: "1:00:00"  # "1:00:00"
-      AutomaticBackupRetentionDays: 1  # 0
-      CopyTagsToBackups: true  # true
-      DailyAutomaticBackupStartTime: 01:03  # 01:03
-      PerUnitStorageThroughput: 50  # 200
+      ImportedFileChunkSize: 1024
+      ExportPath: s3://bucket/folder
+      ImportPath: s3://bucket
+      WeeklyMaintenanceStartTime: "1:00:00"
+      AutomaticBackupRetentionDays: 1
+      CopyTagsToBackups: true
+      DailyAutomaticBackupStartTime: 01:03
+      PerUnitStorageThroughput: 50
       #BackupId: backup-12345678  # BackupId cannot coexist with some of the fields
-      KmsKeyId: String  # xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
-      #FileSystemId: fs-12345678123456789  # FileSystemId cannot coexist with some of the fields
+      KmsKeyId: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+      #FileSystemId: fs-12345678123456789  # FileSystemId cannot coexist with other fields
       AutoImportPolicy: NEW_CHANGED  # NEW | NEW_CHANGED
       DriveCacheType: READ  # READ
       StorageType: HDD  # HDD | SSD
 Iam:
   Roles:
-    CustomLambdaResources: String  # arn:aws:iam::aws:role/CustomResourcesLambdaRole
+    CustomLambdaResources: arn:aws:iam::aws:role/CustomResourcesLambdaRole
 Monitoring:
-  DetailedMonitoring: true  # false
+  DetailedMonitoring: true
   Logs:
     CloudWatch:
-      Enabled: true  # true
-      RetentionInDays: 14  # 14
+      Enabled: true
+      RetentionInDays: 14
       DeletionPolicy: Delete
   Dashboards:
     CloudWatch:
-      Enabled: true  # true
-#AdditionalPackages:
-#  IntelSelectSolutions:
-#    InstallIntelSoftware: true  # false
-#ClusterS3ResourceBucket: String
+      Enabled: true
+CustomS3Bucket: String
 Tags:
   - Key: String
     Value: String
   - Key: two
     Value: two22
-#CustomS3Bucket: String
-#AdditionalResources: String  # https://template.url
-#DevSettings:
+AdditionalResources: https://template.url
+DevSettings:
 #  ClusterTemplate: String
-#  Cookbook:
-#    ChefCookbook: String
-#    ExtraChefAttributes: Object
-#  AwsBatchCliPackage: String
-#  NodePackage: String
+  Cookbook:
+    ChefCookbook: https://template.url
+    ExtraChefAttributes: '{ "cluster" : { "custom_node_package" : "https://s3.eu-west-1.amazonaws.com/cfncluster-fdm-build-artifacts-eu-west-1/node/aws-parallelcluster-node-2.8.1.tgz", "skip_install_recipes": "no" } }'
+  AwsBatchCliPackage: https://batchcli.url
+  NodePackage: https://nodepackage.url

cli/tests/pcluster/schemas/test_cluster_schema/test_cluster_schema_awsbatch/awsbatch.simple.yaml

@@ -1,9 +1,9 @@
 Image:
   Os: alinux2
 HeadNode:
-  InstanceType: String  # t2.micro
+  InstanceType: t2.micro
   Ssh:
-    KeyName: String  # ec2-key-name
+    KeyName: ec2-key-name
   Networking:
     SubnetId: subnet-12345678
 Scheduling:

cli/tests/pcluster/schemas/test_cluster_schema/test_cluster_schema_slurm/slurm.full.yaml

@@ -2,16 +2,16 @@ Image:
   Os: centos7
   CustomAmi: ami-12345678
 HeadNode:
-  InstanceType: t2.micro  # t2.micro
+  InstanceType: t2.micro
   Networking:
-    SubnetId: subnet-12345678  # subnet-xxx
+    SubnetId: subnet-12345678
     ElasticIp: true  # true|false|EIP-id
     AssignPublicIp: True  # true|false
     AdditionalSecurityGroups:
       - sg-34567890
       - sg-45678901
     Proxy:
-      HttpProxyAddress: String  # https://proxy-address:port
+      HttpProxyAddress: https://proxy-address:port
   DisableSimultaneousMultithreading: false
   Ssh:
     KeyName: ec2-key-name
@@ -34,13 +34,13 @@ HeadNode:
     OnNodeStart:
       Script: https://test.tgz  # s3:// | https:// | file://*
       Args:
-        - String  # arg1
-        - stirng2
+        - arg1
+        - arg2
     OnNodeConfigured:
       Script: https://test.tgz  # s3:// | https:// | file://*
       Args:
-        - String  # arg1
-        - stirng2
+        - arg1
+        - arg2
   Iam:
     InstanceRole: arn:aws:iam::aws:role/CustomHeadNodeRole
     S3Access:
@@ -70,13 +70,13 @@ Scheduling:
         OnNodeStart:
           Script: https://test.tgz  # s3:// | https:// | file://*
           Args:
-            - String  # arg1
-            - stirng2
+            - arg1
+            - arg2
         OnNodeConfigured:
           Script: https://test.tgz  # s3:// | https:// | file://*
           Args:
-            - String  # arg1
-            - stirng2
+            - arg1
+            - arg2
       Iam:
         S3Access:
           - BucketName: string1
@@ -104,7 +104,7 @@ Scheduling:
           Enabled: true
           Id: String
         Proxy:
-          HttpProxyAddress: String  # https://proxy-address:port
+          HttpProxyAddress: https://proxy-address:port
       ComputeResources:
         - Name: compute-resource-1
           InstanceType: c4.2xlarge
@@ -146,16 +146,16 @@ SharedStorage:
       DeploymentType: PERSISTENT_1  # PERSISTENT_1 | SCRATCH_1 | SCRATCH_2
       ImportedFileChunkSize: 1024
       DataCompressionType: LZ4
-      ExportPath: String  # s3://bucket/folder
-      ImportPath: String  # s3://bucket
+      ExportPath: s3://bucket/folder
+      ImportPath: s3://bucket
       WeeklyMaintenanceStartTime: "1:00:00"
       AutomaticBackupRetentionDays: 0
       CopyTagsToBackups: true
       DailyAutomaticBackupStartTime: 01:03
       PerUnitStorageThroughput: 200
       # BackupId: backup-fedcba98 # BackupId cannot coexist with some of the fields
-      KmsKeyId: String  # xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
-      # FileSystemId: fs-12345678123456789 # FileSystemId cannot coexist with some of the fields
+      KmsKeyId: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+      # FileSystemId: fs-12345678123456789 # FileSystemId cannot coexist with other fields
       AutoImportPolicy: NEW  # NEW | NEW_CHANGED
       DriveCacheType: READ  # READ
       StorageType: HDD  # HDD | SSD
@@ -181,7 +181,7 @@ Tags:
   - Key: two
     Value: two22
 CustomS3Bucket: String
-AdditionalResources: String  # https://template.url
+AdditionalResources: https://template.url
 DevSettings:
   ClusterTemplate: file:///tests/aws-parallelcluster-template-3.0.tgz
   Cookbook: